Skip to content

2023-12目前项目已经不能使用 #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MisterChangRay wants to merge 3 commits into
base: master
Choose a base branch
from
Open

2023-12目前项目已经不能使用 #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
16354f3
upgrade
MisterChangRay Dec 22, 2023
7fba793
upgrade date parser
MisterChangRay Dec 22, 2023
9ce8a2f
remove print
MisterChangRay Dec 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,5 +450,12 @@ title中可带有单引号。

爬虫保存的markdown文件中的图片链接有些为http协议的，jekyll生成的页面中这种图片有些会显示不正常，所以图片链接最好改为http协议。程序中已经将图片链接中的https替换为了http。

#### 6、windows命令行乱码

执行`chcp 65001`


#### 7、执行报错`LookupError: unknown encoding: cp65001`

cmd执行报错`LookupError: unknown encoding: cp65001`， 执行`set PYTHONIOENCODING=UTF-8`

45 changes: 27 additions & 18 deletions export_csdn_markdown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python2.7
# coding=utf-8


import re
import os # 多种操作系统接口：该模块提供了一种使用操作系统相关功能的便携方式。
import sys # 系统特定的参数和功能：该模块提供对解释器使用或维护的一些变量的访问，以及与解释器强烈交互的函数。
# 在程序中加入以下代码：将编码设置为utf8
Expand Down Expand Up @@ -51,6 +51,7 @@ def CrawlingItemBlog(base_url, id):
url = second_url + id
# 发送request请求并接受返回值
item_html = request_get(url)
print("---", url)
if item_html.status_code == 200: # 200说明request_get完成，这是因为http协议里面定义的状态码
'''
需要的信息：
Expand Down Expand Up @@ -82,7 +83,7 @@ def CrawlingItemBlog(base_url, id):
title_article = soup.find(attrs={'class': 'title-article'})
# 这里是将标题作为最后存储的文件名
file_name = title_article.get_text()
print(" 该篇博客标题为：" + file_name)
print(" 该篇博客标题为：" + file_name.encode("utf-8"))
title_article = title_article.prettify()

# 设置jekyll格式博客开头的格式（title）
Expand All @@ -105,14 +106,18 @@ def CrawlingItemBlog(base_url, id):
jekyll_categories = 'categories:\n' + '- ' + jekyll_categories + '\n'

# 获取文章发表时间
time = soup.find(attrs={'class': 'time'}).get_text()
s_time1 = time.split('年')

time = soup.find(attrs={'class': 'time'}).text

time = re.match(".*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*", time).group(1)
s_time0 = time.split(' ')
s_time1 = s_time0[0].split('-')


year = s_time1[0]
s_time2 = s_time1[1].split('月')
month = s_time2[0]
s_time3 = s_time2[1].split('日')
day = s_time3[0]
minite = s_time3[1].strip()
month = s_time1[1]
day = s_time1[2]
minite = s_time0[1].split(':')[1]

jekyll_date = 'date: ' + year + '-' + month + '-' + day + ' ' + minite + '\n'

Expand Down Expand Up @@ -158,9 +163,9 @@ def CrawlingItemBlog(base_url, id):
# 有的文章名字特殊，会新建文件失败
try:
# 写入文件
md_file_name = './_posts/' + year + '-' + month + '-' + day + '-' + file_name + '.markdown'
md_file_name = './_posts/' + year + '-' + month + '-' + day + '-' + file_name + '.md'
f = codecs.open(md_file_name, 'w', encoding='utf-8')
jekyll_str = '---\n' + 'layout: post\n' + jekyll_title + jekyll_date + 'author: "唐传林"\nheader-img: "img/post-bg-2015.jpg"\ncatalog: false\n' + jekyll_categories + jekyll_tags + '\n---\n'
jekyll_str = '---\n' + 'layout: post\n' + jekyll_title + jekyll_date + 'author: "ray.zh"\nheader-img: "img/post-bg-2015.jpg"\ncatalog: false\n' + jekyll_categories + jekyll_tags + '\n---\n'

f.write(jekyll_str)
f.write(text_utf8_right)
Expand Down Expand Up @@ -230,22 +235,26 @@ def start_spider(username):

number = 1 # 记录当前是第几个article_list
count = 0 # 记录当前是第几篇文章

# print(start_url)
# 开始爬取第一个article_list，返回信息在html中
html = request_get(start_url)

# 这个循环是对博客的article_list页面的循环
while html.status_code == 200: # 200说明request_get完成，这是因为http协议里面定义的状态码
# 获取下一页的 url
selector = etree.HTML(html.text)

#print(html.text.encode("utf-8"))
# cur_article_list_page[0]就是当前article_list页面中的文章的list
cur_article_list_page = selector.xpath('//*[@id="mainBox"]/main/div[2]')
d = cur_article_list_page[0].xpath('//*[@id="mainBox"]/main/div[2]/div[2]/h4/a')
l = cur_article_list_page[0].findall('data-articleid')
cur_article_list_page = selector.xpath('//*[@id="articleMeList-blog"]/div[2]/div')
# cur_article_list_page = '//*[@id="articleMeList-blog"]/div[2]/div'
#d = selector2.xpath('//*[@id="mainBox"]/main/div[2]/div[2]/div/h4/a')
#print(d)
# l = selector.findall('[data-articleid]')
#l = selector3.xpath('//*[@id="articleMeList-blog"]/div[2]/div')

# 这个循环是对你每一个article_list中的那些文章的循环
for elem in cur_article_list_page[0]:
for elem in cur_article_list_page:

item_content = elem.attrib
# 通过对比拿到的数据和网页中的有效数据发现返回每一个article_list中的list都有一两个多余元素，每个多余元素都有style属性，利用这一特点进行过滤
if item_content.has_key('style'):
Expand All @@ -267,7 +276,7 @@ def start_spider(username):


if __name__ == "__main__":
username = 'Tang_Chuanlin' # CSDN 个人主页地址中的用户名，例如https://blog.csdn.net/Tang_Chuanlin中的Tang_Chuanlin
username = 'jioulongzi' # CSDN 个人主页地址中的用户名，例如https://blog.csdn.net/Tang_Chuanlin中的Tang_Chuanlin
if os.path.exists('./imgs/'): # 判断当前路径下是否存在"imgs"文件夹
shutil.rmtree('./imgs/') # 若存在，则删除该文件夹，目的是删除之前爬取获得的图片
mkdir('./imgs/') # "imgs"新建文件夹
Expand Down