108 lines
4.7 KiB
Python
108 lines
4.7 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""P3-MD: Markdown language inventory classifier (documentation-only, read-only).
|
||
|
|
|
||
|
|
Counts Indonesian vs English stopword/prose markers per .md file and classifies:
|
||
|
|
ENGLISH / PARTIALLY ENGLISH / NON-ENGLISH / CODE-ONLY
|
||
|
|
Writes inventory JSON (path, tree, size, language, score, status).
|
||
|
|
"""
|
||
|
|
import os
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
|
||
|
|
TREES = {
|
||
|
|
"FORGE_REPO": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\AlgoForge\publish\AlgoForge",
|
||
|
|
"WORKING_COPY": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\AlgoForge",
|
||
|
|
"LEGACY": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\SniperGold_ML",
|
||
|
|
}
|
||
|
|
|
||
|
|
ID = [
|
||
|
|
"yang", "dan", "untuk", "dengan", "pada", "dari", "tidak", "adalah", "ini",
|
||
|
|
"akan", "telah", "dapat", "harus", "bahwa", "atau", "sebagai", "setelah",
|
||
|
|
"sebelum", "karena", "namun", "tetapi", "antara", "terhadap", "dilakukan",
|
||
|
|
"menggunakan", "terdapat", "menjadi", "berdasarkan", "sesuai", "terkait",
|
||
|
|
"melalui", "tanpa", "sudah", "juga", "masih", "lebih", "sangat", "berikut",
|
||
|
|
"sekitar", "hingga", "sampai", "semua", "beberapa", "setiap", "saat",
|
||
|
|
"ketika", "jika", "misalnya", "contoh", "catatan", "kesimpulan", "temuan",
|
||
|
|
"hasil", "tujuan", "bukti", "perlu", "bila", "bukan", "seluruh", "dalam",
|
||
|
|
"kembali", "diubah", "diperbaiki", "terverifikasi", "diverifikasi",
|
||
|
|
"dokumentasi", "penelitian", "sesi", "tanggal", "status", "disimpulkan",
|
||
|
|
"karena", "wajib", "jangan", "silahkan", "lihat", "pakai", "memakai",
|
||
|
|
"sama", "berbeda", "dibuat", "dihitung", "dipakai", "utk", "thd", "dgn",
|
||
|
|
"pd", "utk", "sbg", "tdk", "krn", "sbb", "dst", "dll", "yaitu", "yakni",
|
||
|
|
"tersebut", "tersebut", "berikutnya", "sebelumnya", "kini", "saat ini",
|
||
|
|
"awal", "akhir", "langkah", "bagian", "bab", "subbab", "ringkasan",
|
||
|
|
"lampiran", "catatan", "terlampir", "pengujian", "pengujian", "uji",
|
||
|
|
"kasus", "sampel", "populasi", "rerata", "median", "persentase", "rata",
|
||
|
|
"grafik", "tabel", "gambar", "laporan", "handover", "checkpoint",
|
||
|
|
]
|
||
|
|
EN = [
|
||
|
|
"the", "and", "for", "with", "this", "that", "from", "not", "are", "will",
|
||
|
|
"have", "has", "can", "must", "should", "between", "against", "using",
|
||
|
|
"based", "according", "through", "without", "also", "still", "more",
|
||
|
|
"following", "around", "until", "all", "some", "every", "when", "if",
|
||
|
|
"example", "note", "conclusion", "finding", "result", "objective",
|
||
|
|
"evidence", "verified", "confirmed", "documentation", "research",
|
||
|
|
"session", "date", "status", "concluded", "summary", "appendix",
|
||
|
|
"testing", "test", "case", "sample", "population", "median", "table",
|
||
|
|
"figure", "report", "analysis", "audit", "specification", "conformance",
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def classify(path):
|
||
|
|
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||
|
|
text = f.read()
|
||
|
|
# strip code fences
|
||
|
|
no_code = re.sub(r"```.*?```", " ", text, flags=re.S)
|
||
|
|
# strip inline code
|
||
|
|
no_code = re.sub(r"`[^`]*`", " ", no_code)
|
||
|
|
words = re.findall(r"[A-Za-z]{3,}", no_code.lower())
|
||
|
|
n_id = sum(1 for w in words if w in ID)
|
||
|
|
n_en = sum(1 for w in words if w in EN)
|
||
|
|
total = len(words)
|
||
|
|
score = (n_id - n_en) / max(1, total) * 1000.0 # per-mille
|
||
|
|
if total < 30:
|
||
|
|
lang = "CODE-ONLY" if score < 5 else "NON-ENGLISH"
|
||
|
|
elif score <= 2:
|
||
|
|
lang = "ENGLISH"
|
||
|
|
elif score <= 8:
|
||
|
|
lang = "PARTIALLY ENGLISH"
|
||
|
|
else:
|
||
|
|
lang = "NON-ENGLISH"
|
||
|
|
return lang, score, total, n_id, n_en
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
rows = []
|
||
|
|
for tree, root in TREES.items():
|
||
|
|
for dirpath, _dirs, files in os.walk(root):
|
||
|
|
if ".git" in dirpath:
|
||
|
|
continue
|
||
|
|
for fn in files:
|
||
|
|
if fn.lower().endswith(".md"):
|
||
|
|
p = os.path.join(dirpath, fn)
|
||
|
|
size = os.path.getsize(p)
|
||
|
|
lang, score, total, nid, nen = classify(p)
|
||
|
|
rows.append({
|
||
|
|
"tree": tree,
|
||
|
|
"path": os.path.relpath(p, root),
|
||
|
|
"size": size,
|
||
|
|
"language": lang,
|
||
|
|
"score_permille": round(score, 2),
|
||
|
|
"words": total,
|
||
|
|
"id_words": nid,
|
||
|
|
"en_words": nen,
|
||
|
|
})
|
||
|
|
rows.sort(key=lambda r: (r["tree"], r["path"]))
|
||
|
|
out = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
|
|
"..", "..", "output", "md_language_inventory.json")
|
||
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||
|
|
with open(out, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(rows, f, indent=2)
|
||
|
|
for r in rows:
|
||
|
|
print(f"{r['tree']:<14} {r['language']:<18} {r['score_permille']:>8} "
|
||
|
|
f"{r['path']}")
|
||
|
|
print(f"\nTOTAL {len(rows)} files -> {out}")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|