如何写一个爬虫-批量抓取网页数据

**不懂爬虫是啥，就自己百度啦

首次写头条号，据说可以自动推送特定用户，就试试，首次就送上一个现成品吧！

不多说，直接开始！

要爬的网址：http://www.shengzhujiage.com/zjg/mao/

脚本：get_from_web_Useful.py

备注，除了第一句，其他#开头的为注释，看思路的可以好好看注释！

1. # -*- coding: UTF-8 -*-

2. #usage: sys.argv[0] url output

3. # http://www.shengzhujiage.com/zjg/mao/

5. import sys

6. import urllib

7. import urllib2

8. import re

9. import os

11. ########################

12. ##### Functions #####

13. ########################

14. def getHtml(url):

15. request = urllib2.Request(url)

16. respones = urllib2.urlopen(request)

17. html = respones.read()

18. return html

20. def getLinks(html):

21. #get the each day link

22. ms1 = u'毛猪价格</a>]·'.encode('CP936')

23. ms2 = u'[0-9\u4e00-\u9fa5]+全国毛猪[\u4e00-\u9fa5]+'.encode('CP936')

24. r1 = r'%s<a href=\'([\w|\s|\d|.:/]+)\'>%s'%(ms1,ms2)

25. r1_compile = re.compile(r1)

26. link_list = re.findall(r1_compile,html)

27. return link_list

29. def replace(x):

30. re_Br = re.compile('<br />|<br>')

31. re_Addr = re.compile('<a.*?>|</a>')

32. re_tab = re.compile('[ ]+')

33. x = re.sub(re_Addr,"",x)

34. x = re.sub(re_Br,"\n",x)

35. x = re.sub(re_tab,"\t",x)

36. return x

38. def getInfo1(subHtml):

39. #get the subHtml info

40. r1 = r'data"(.+)<\/div>'

41. sub_re = re.compile(r1)

42. r1_result = re.findall(sub_re,subHtml)

43. return r1_result

45. def getTitle(subHtml):

46. #get the subHtml title

47. patten = re.compile('<title>(.*)?</title>')

48. result = re.findall(patten,subHtml)

49. return result

51. def makeDir(path):

52. if os.path.exists(path):

53. print "[INFO]",path,u"路径存在"

54. else:

55. print u"[INFO] 具体数据将保存在目录",path,u"中！"

56. print u"[INFO] 正在创建目录：",path

57. os.makedirs(path)

59. ########################

60. ##### Main #####

61. ########################

62. html = getHtml(sys.argv[1])#print html

63. #get the each day link

64. list = getLinks(html)

65. #make the directory

66. path = r'D:/PIG_PRICE'#change the dir_path as you like

67. makeDir(path)

69. #get useful info from each link

70. for link in list:

71. print u"[INFO] 当前链接为："+link

72. subHtml = getHtml(link)

73. title = getTitle(subHtml)

74. filename = path+'/'+title[0]+".xls"

75. #print filename

76. if os.path.exists(filename):

77. print '[INFO]',filename,u'文件存在'

78. continue

79. else:

80. print u'[INFO] 创建文件：',filename

81. Info= getInfo1(subHtml)

82. out=open(filename,"w")

83. for info inInfo:

84. info = replace(info)

85. #print info

86. out.write(info)

87. print u"[DONE] 已获取到",title[0]+".xls",u"数据"

88. out.close()

89. print u"[FINISH] 完成链接：",sys.argv[1],u"中的所有日期"

90. #print

运行：pythonget_from_web_Useful.pyhttp://www.shengzhujiage.com/zjg/mao/

其中http://www.shengzhujiage.com/zjg/mao/ 可被替换

将这里面的每一页的链接获取后替换，运行