SniperGold_ML/ml/p3/smc_semantic/md_language_inventory.py

# -*- coding: utf-8 -*-
"""P3-MD: Markdown language inventory classifier (documentation-only, read-only).

Counts Indonesian vs English stopword/prose markers per .md file and classifies:
  ENGLISH / PARTIALLY ENGLISH / NON-ENGLISH / CODE-ONLY
Writes inventory JSON (path, tree, size, language, score, status).
"""
import os
import json
import re

TREES = {
    "FORGE_REPO": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\AlgoForge\publish\AlgoForge",
    "WORKING_COPY": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\AlgoForge",
    "LEGACY": r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Shared Projects\SniperGold_ML",
}

ID = [
    "yang", "dan", "untuk", "dengan", "pada", "dari", "tidak", "adalah", "ini",
    "akan", "telah", "dapat", "harus", "bahwa", "atau", "sebagai", "setelah",
    "sebelum", "karena", "namun", "tetapi", "antara", "terhadap", "dilakukan",
    "menggunakan", "terdapat", "menjadi", "berdasarkan", "sesuai", "terkait",
    "melalui", "tanpa", "sudah", "juga", "masih", "lebih", "sangat", "berikut",
    "sekitar", "hingga", "sampai", "semua", "beberapa", "setiap", "saat",
    "ketika", "jika", "misalnya", "contoh", "catatan", "kesimpulan", "temuan",
    "hasil", "tujuan", "bukti", "perlu", "bila", "bukan", "seluruh", "dalam",
    "kembali", "diubah", "diperbaiki", "terverifikasi", "diverifikasi",
    "dokumentasi", "penelitian", "sesi", "tanggal", "status", "disimpulkan",
    "karena", "wajib", "jangan", "silahkan", "lihat", "pakai", "memakai",
    "sama", "berbeda", "dibuat", "dihitung", "dipakai", "utk", "thd", "dgn",
    "pd", "utk", "sbg", "tdk", "krn", "sbb", "dst", "dll", "yaitu", "yakni",
    "tersebut", "tersebut", "berikutnya", "sebelumnya", "kini", "saat ini",
    "awal", "akhir", "langkah", "bagian", "bab", "subbab", "ringkasan",
    "lampiran", "catatan", "terlampir", "pengujian", "pengujian", "uji",
    "kasus", "sampel", "populasi", "rerata", "median", "persentase", "rata",
    "grafik", "tabel", "gambar", "laporan", "handover", "checkpoint",
]
EN = [
    "the", "and", "for", "with", "this", "that", "from", "not", "are", "will",
    "have", "has", "can", "must", "should", "between", "against", "using",
    "based", "according", "through", "without", "also", "still", "more",
    "following", "around", "until", "all", "some", "every", "when", "if",
    "example", "note", "conclusion", "finding", "result", "objective",
    "evidence", "verified", "confirmed", "documentation", "research",
    "session", "date", "status", "concluded", "summary", "appendix",
    "testing", "test", "case", "sample", "population", "median", "table",
    "figure", "report", "analysis", "audit", "specification", "conformance",
]


def classify(path):
    with open(path, "r", encoding="utf-8", errors="replace") as f:
        text = f.read()
    # strip code fences
    no_code = re.sub(r"```.*?```", " ", text, flags=re.S)
    # strip inline code
    no_code = re.sub(r"`[^`]*`", " ", no_code)
    words = re.findall(r"[A-Za-z]{3,}", no_code.lower())
    n_id = sum(1 for w in words if w in ID)
    n_en = sum(1 for w in words if w in EN)
    total = len(words)
    score = (n_id - n_en) / max(1, total) * 1000.0  # per-mille
    if total < 30:
        lang = "CODE-ONLY" if score < 5 else "NON-ENGLISH"
    elif score <= 2:
        lang = "ENGLISH"
    elif score <= 8:
        lang = "PARTIALLY ENGLISH"
    else:
        lang = "NON-ENGLISH"
    return lang, score, total, n_id, n_en


def main():
    rows = []
    for tree, root in TREES.items():
        for dirpath, _dirs, files in os.walk(root):
            if ".git" in dirpath:
                continue
            for fn in files:
                if fn.lower().endswith(".md"):
                    p = os.path.join(dirpath, fn)
                    size = os.path.getsize(p)
                    lang, score, total, nid, nen = classify(p)
                    rows.append({
                        "tree": tree,
                        "path": os.path.relpath(p, root),
                        "size": size,
                        "language": lang,
                        "score_permille": round(score, 2),
                        "words": total,
                        "id_words": nid,
                        "en_words": nen,
                    })
    rows.sort(key=lambda r: (r["tree"], r["path"]))
    out = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                       "..", "..", "output", "md_language_inventory.json")
    os.makedirs(os.path.dirname(out), exist_ok=True)
    with open(out, "w", encoding="utf-8") as f:
        json.dump(rows, f, indent=2)
    for r in rows:
        print(f"{r['tree']:<14} {r['language']:<18} {r['score_permille']:>8} "
              f"{r['path']}")
    print(f"\nTOTAL {len(rows)} files -> {out}")


if __name__ == "__main__":
    main()