shaochangli/articleCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update start.py

Browse Source 
修正了若干常见问题
This commit is contained in:
LeLe86 2022-03-19 20:00:36 +08:00 committed by GitHub
parent 0d52e92361
commit ac3473fda1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 46 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
start.py
View File

@ -8,8 +8,7 @@ from time import sleep
""" """
本项目开源地址 https://github.com/LeLe86/vWeChatCrawl 本项目开源地址 https://github.com/LeLe86/vWeChatCrawl
讨论QQ群 703431832 讨论QQ群 703431832 加群暗号:不止技术流
""" """
#保存文件 #保存文件
@ -51,8 +50,9 @@ def DownLoadHtml(url):
'Connection':'keep-alive', 'Connection':'keep-alive',
'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3' 'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
} }
requests.packages.urllib3.disable_warnings() session = requests.Session()
response = requests.get(url,headers = headers,proxies=None,verify=False) session.trust_env = False
response = session.get(url,headers = headers)
if response.status_code == 200: if response.status_code == 200:
htmltxt = response.text #返回的网页正文 htmltxt = response.text #返回的网页正文
return htmltxt return htmltxt
@ -68,10 +68,11 @@ def DownImg(url,savepath):
'Connection':'keep-alive', 'Connection':'keep-alive',
'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3' 'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
} }
requests.packages.urllib3.disable_warnings() session = requests.Session()
r = requests.get(url,headers = headers,proxies=None,verify=False) session.trust_env = False
response = session.get(url, headers=headers)
with open(savepath, 'wb') as f: with open(savepath, 'wb') as f:
f.write(r.content) f.write(response.content)
#修改网页中图片的src，使图片能正常显示 #修改网页中图片的src，使图片能正常显示
def ChangeImgSrc(htmltxt,saveimgdir,htmlname): def ChangeImgSrc(htmltxt,saveimgdir,htmlname):
@ -89,7 +90,7 @@ def ChangeImgSrc(htmltxt,saveimgdir,htmlname):
originalURL = "" originalURL = ""
if originalURL.startswith("//"):#如果url以//开头，则需要添加http： if originalURL.startswith("//"):#如果url以//开头，则需要添加http：
originalURL = "http:" + originalURL originalURL = "http:" + originalURL
if len(originalURL) > 0: if len(originalURL) > 20:
print("\r down imgs " + "▇" * imgindex +" " + str(imgindex),end="") print("\r down imgs " + "▇" * imgindex +" " + str(imgindex),end="")
if "data-type" in img.attrs: if "data-type" in img.attrs:
imgtype = img.attrs["data-type"] imgtype = img.attrs["data-type"]
@ -133,6 +134,7 @@ def GetArticleList(jsondir):
filelist = os.listdir(jsondir) filelist = os.listdir(jsondir)
ArtList = [] ArtList = []
for file in filelist: for file in filelist:
try:
filepath = os.path.join(jsondir,file) filepath = os.path.join(jsondir,file)
filetxt = ReadFile(filepath) filetxt = ReadFile(filepath)
jsbody = json.loads(filetxt) jsbody = json.loads(filetxt)
@ -165,6 +167,8 @@ def GetArticleList(jsondir):
if len(url)>3: if len(url)>3:
ArtList.append(art) ArtList.append(art)
print(len(ArtList),pubdate, idx, title) print(len(ArtList),pubdate, idx, title)
except:
print("跳过，可不用管",file)
return ArtList return ArtList
def DownHtmlMain(jsonDir,saveHtmlDir): def DownHtmlMain(jsonDir,saveHtmlDir):