186 lines
7.9 KiB
Python
 | '''Сохраняет опубликованные части 1–3: текст, HTML и изображения.
| ||
| |||
Запуск из корня: python docs/articles/import_published.py
| |||
Повторный запуск обновляет локальные копии с сайта.
| |||
Используется только стандартная библиотека Python.
| |||
'''
| |||
| |||
import hashlib
| |||
import html
| |||
import re
| |||
from datetime import datetime, timedelta, timezone
| |||
from html.parser import HTMLParser
| |||
from pathlib import Path
| |||
from urllib.parse import urljoin, urlparse
| |||
from urllib.request import Request, urlopen
| |||
| |||
ARTICLES = {1: 19804, 2: 19852, 3: 19946}
| |||
ROOT = Path(__file__).resolve().parent
| |||
VOID = {'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'wbr'}
| |||
| |||
| |||
class Node:
| |||
def __init__(self, tag='', attrs=()):
| |||
self.tag, self.attrs, self.children = tag, dict(attrs), []
| |||
| |||
def find(self, predicate):
| |||
if predicate(self):
| |||
return self
| |||
for child in self.children:
| |||
if isinstance(child, Node):
| |||
found = child.find(predicate)
| |||
if found is not None:
| |||
return found
| |||
return None
| |||
| |||
def text(self):
| |||
return ''.join(child.text() if isinstance(child, Node) else child for child in self.children)
| |||
| |||
| |||
class Parser(HTMLParser):
| |||
def __init__(self):
| |||
super().__init__(convert_charrefs=True)
| |||
self.root = Node()
| |||
self.stack = [self.root]
| |||
| |||
def handle_starttag(self, tag, attrs):
| |||
node = Node(tag, attrs)
| |||
self.stack[-1].children.append(node)
| |||
if tag not in VOID:
| |||
self.stack.append(node)
| |||
| |||
def handle_startendtag(self, tag, attrs):
| |||
self.handle_starttag(tag, attrs)
| |||
if tag not in VOID:
| |||
self.handle_endtag(tag)
| |||
| |||
def handle_endtag(self, tag):
| |||
for index in range(len(self.stack) - 1, 0, -1):
| |||
if self.stack[index].tag == tag:
| |||
del self.stack[index:]
| |||
break
| |||
| |||
def handle_data(self, data):
| |||
self.stack[-1].children.append(data)
| |||
| |||
| |||
def download(url):
| |||
with urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0'}), timeout=45) as response:
| |||
return response.read()
| |||
| |||
| |||
def serialize(node):
| |||
if isinstance(node, str):
| |||
return html.escape(node, quote=False)
| |||
if node.tag in ('script', 'style'):
| |||
return ''
| |||
attrs = ''.join(f' {key}="{html.escape(value or "", quote=True)}"' for key, value in node.attrs.items())
| |||
body = ''.join(serialize(child) for child in node.children)
| |||
if not node.tag:
| |||
return body
| |||
return f'<{node.tag}{attrs}>' + ('' if node.tag in VOID else body + f'</{node.tag}>')
| |||
| |||
| |||
def markdown(node):
| |||
if isinstance(node, str):
| |||
return re.sub(r'\s+', ' ', node)
| |||
tag = node.tag
| |||
if tag in ('script', 'style'):
| |||
return ''
| |||
if tag == 'pre':
| |||
code = node.text().replace('\xa0', ' ').strip('\r\n')
| |||
fence = '`' * max(3, max((len(m.group()) + 1 for m in re.finditer(r'`+', code)), default=3))
| |||
return f'\n\n{fence}\n{code}\n{fence}\n\n'
| |||
if tag == 'table':
| |||
return '\n\n' + serialize(node) + '\n\n'
| |||
body = ''.join(markdown(child) for child in node.children)
| |||
if re.fullmatch(r'h[1-6]', tag):
| |||
return '\n\n' + '#' * max(2, int(tag[1]) - 1) + ' ' + body.strip() + '\n\n'
| |||
if tag == 'a' and node.attrs.get('href'):
| |||
return f'[{body.strip()}](<{node.attrs["href"]}>)'
| |||
if tag == 'img':
| |||
return f'![{node.attrs.get("alt", "")}](<{node.attrs.get("src", "")}>)'
| |||
if tag in ('b', 'strong'):
| |||
return '**' + body.strip() + '**' if body.strip() else ''
| |||
if tag in ('i', 'em'):
| |||
return '*' + body.strip() + '*' if body.strip() else ''
| |||
if tag == 'code':
| |||
return '`' + body.strip() + '`'
| |||
if tag == 'li':
| |||
return '\n- ' + body.strip().replace('\n\n', '\n ') + '\n'
| |||
if tag == 'br':
| |||
return ' \n'
| |||
if tag in ('p', 'div', 'section', 'ul', 'ol', 'blockquote'):
| |||
return '\n\n' + body.strip() + '\n\n'
| |||
return body
| |||
| |||
| |||
def clean_markdown(text):
| |||
lines, in_code = [], False
| |||
for line in text.replace('\r', '').splitlines():
| |||
if line.lstrip().startswith('```'):
| |||
in_code = not in_code
| |||
# Сохраняем жёсткие переносы Markdown без хвостовых пробелов.
| |||
hard_break = not in_code and line.endswith(' ') and bool(line.strip())
| |||
lines.append(line.rstrip() + ('<br>' if hard_break else ''))
| |||
return '\n'.join(lines)
| |||
| |||
| |||
def save(part, article_id):
| |||
url = f'https://www.mql5.com/ru/articles/{article_id}'
| |||
parser = Parser()
| |||
parser.feed(download(url).decode('utf-8-sig').replace('\r', ''))
| |||
title_node = parser.root.find(lambda node: node.tag == 'h1')
| |||
content = parser.root.find(lambda node: node.tag == 'div' and
| |||
{'inner', 'formatted-content'}.issubset(node.attrs.get('class', '').split()))
| |||
if title_node is None or content is None or len(content.text()) < 3000:
| |||
raise RuntimeError(f'Не найден полный текст статьи {article_id}')
| |||
title = title_node.text().strip()
| |||
folder = ROOT / f'part-{part:02d}'
| |||
(folder / 'images').mkdir(parents=True, exist_ok=True)
| |||
(folder / 'publish').mkdir(exist_ok=True)
| |||
counts = {'images': 0, 'code': 0}
| |||
| |||
def localize(node):
| |||
if not isinstance(node, Node):
| |||
return
| |||
if node.tag == 'pre':
| |||
counts['code'] += 1
| |||
if node.attrs.get('href'):
| |||
node.attrs['href'] = urljoin(url, node.attrs['href'])
| |||
if node.tag == 'img' and node.attrs.get('src'):
| |||
source = urljoin(url, node.attrs['src'])
| |||
if urlparse(source).hostname == 'c.mql5.com':
| |||
name = hashlib.sha256(source.encode()).hexdigest()[:8] + '-' + Path(urlparse(source).path).name
| |||
(folder / 'images' / name).write_bytes(download(source))
| |||
node.attrs['src'] = 'images/' + name
| |||
counts['images'] += 1
| |||
else:
| |||
node.attrs['src'] = source
| |||
for child in node.children:
| |||
localize(child)
| |||
| |||
localize(content)
| |||
date = datetime.now(timezone(timedelta(hours=3))).date().isoformat()
| |||
text = clean_markdown(markdown(content))
| |||
text = re.sub(r'(?m)^[ \t]+$', '', text)
| |||
text = re.sub(r'\n{3,}', '\n\n', text).strip()
| |||
header = f'# {title}\n\nИсточник: [{url}]({url}). Локальная копия от {date}.\n\n'
| |||
(folder / 'article.md').write_text(header + text + '\n', encoding='utf-8', newline='\n')
| |||
fragment = serialize(content).replace('src="images/', 'src="../images/')
| |||
fragment = '\n'.join(line.rstrip() for line in fragment.replace('\r', '').splitlines())
| |||
(folder / 'publish/article.html').write_text(
| |||
'<!doctype html>\n<meta charset="utf-8">\n<h1>' + html.escape(title) + '</h1>\n' + fragment + '\n', encoding='utf-8', newline='\n')
| |||
(folder / 'source.md').write_text(f'# Источник\n\n- Статья: [{title}]({url}).\n'
| |||
f'- Автор: Yuriy Bykov.\n- Дата загрузки: {date}.\n'
| |||
f'- Блоков кода: {counts["code"]}; изображений: {counts["images"]}.\n\n'
| |||
'Markdown получен из полного текста страницы. Таблицы сохранены в HTML, '
| |||
'листинги — отдельными блоками, изображения — локально. '
| |||
'Исходное форматирование тела статьи сохранено в `publish/article.html`; '
| |||
'навигация сайта, реклама и обсуждение не включены.\n', encoding='utf-8')
| |||
print(f'part-{part:02d}: {len(text)} символов, {counts}')
| |||
| |||
| |||
if __name__ == '__main__':
| |||
for part, article_id in ARTICLES.items():
| |||
save(part, article_id)
|