shaochangli/articleCrawler
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

自动兼容json文件路径

Browse Source
This commit is contained in:
LeLe 2019-08-23 15:19:23 +08:00
parent 7b0f68c08c
commit 7d7c629ae8
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.json
View File

@ -1,5 +1,5 @@
{ {
"jsonDir": "C:/vWeChatFiles/rawlist/Dump-0805-15-00-45", "jsonDir": "C:/vWeChatFiles/rawlist/tmp1",
"htmlDir": "c:/vWeChatFiles/html/", "htmlDir": "c:/vWeChatFiles/html/",
"pdfDir": "c:/vWeChatFiles/pdf/" "pdfDir": "c:/vWeChatFiles/pdf/"
} }

5
start.py
View File

@ -32,6 +32,7 @@ def Timestamp2Datetime(stampstr):
#初始化环境 #初始化环境
def GetJson(): def GetJson():
jstxt = ReadFile("config.json") jstxt = ReadFile("config.json")
jstxt = jstxt.replace("\\\\","/").replace("\\","/") #防止json中有 / 导致无法识别
jsbd = json.loads(jstxt) jsbd = json.loads(jstxt)
return jsbd return jsbd
@ -46,7 +47,7 @@ def DownLoadHtml(url):
'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3' 'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
} }
response = requests.get(url,headers = headers) response = requests.get(url,headers = headers,proxies=None)
if response.status_code == 200: if response.status_code == 200:
htmltxt = response.text #返回的网页正文 htmltxt = response.text #返回的网页正文
return htmltxt return htmltxt
@ -62,7 +63,7 @@ def DownImg(url,savepath):
'Connection':'keep-alive', 'Connection':'keep-alive',
'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3' 'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
} }
r = requests.get(url,headers = headers) r = requests.get(url,headers = headers,proxies=None)
with open(savepath, 'wb') as f: with open(savepath, 'wb') as f:
f.write(r.content) f.write(r.content)