From 05429935948299e25b31fbee0c47282be123d4fd Mon Sep 17 00:00:00 2001 From: Oluwarotimi Quadri Date: Tue, 23 Sep 2025 20:23:20 -0400 Subject: [PATCH] scoring pipeline update --- .coverage | Bin 53248 -> 0 bytes .gitignore | 4 + requirments.txt => requirements.txt | 0 run | 4 + src/acmecli.egg-info/SOURCES.txt | 11 +- .../__pycache__/__init__.cpython-313.pyc | Bin 144 -> 0 bytes src/acmecli/__pycache__/types.cpython-313.pyc | Bin 4449 -> 0 bytes src/acmecli/cache.py | 19 ++ src/acmecli/cli.py | 149 ++++++++++-- src/acmecli/handlers/__init__.py | 3 + src/acmecli/handlers/github.py | 145 ++++++++++++ src/acmecli/metrics/__init__.py | 21 ++ .../__pycache__/__init__.cpython-313.pyc | Bin 152 -> 0 bytes .../metrics/__pycache__/base.cpython-313.pyc | Bin 591 -> 0 bytes .../license_metric.cpython-313.pyc | Bin 1364 -> 0 bytes src/acmecli/metrics/base.py | 57 ++++- src/acmecli/metrics/heuristic_metrics.py | 213 ++++++++++++++++++ src/acmecli/metrics/license_metric.py | 43 +++- src/acmecli/types.py | 54 +++-- ...rics_contract.cpython-313-pytest-8.4.2.pyc | Bin 3326 -> 0 bytes ...porter_schema.cpython-313-pytest-8.4.2.pyc | Bin 2003 -> 0 bytes tests/test_cli_flow.py | 97 ++++++++ tests/test_collect_all.py | 70 ++++++ tests/test_handlers_github.py | 81 +++++++ tests/test_metrics_behavior.py | 154 +++++++++++++ tests/test_reporter_schema.py | 32 +++ 26 files changed, 1117 insertions(+), 40 deletions(-) delete mode 100644 .coverage create mode 100644 .gitignore rename requirments.txt => requirements.txt (100%) delete mode 100644 src/acmecli/__pycache__/__init__.cpython-313.pyc delete mode 100644 src/acmecli/__pycache__/types.cpython-313.pyc create mode 100644 src/acmecli/cache.py create mode 100644 src/acmecli/handlers/__init__.py create mode 100644 src/acmecli/handlers/github.py delete mode 100644 src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc delete mode 100644 src/acmecli/metrics/__pycache__/base.cpython-313.pyc delete mode 100644 src/acmecli/metrics/__pycache__/license_metric.cpython-313.pyc create mode 100644 src/acmecli/metrics/heuristic_metrics.py delete mode 100644 tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc delete mode 100644 tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc create mode 100644 tests/test_cli_flow.py create mode 100644 tests/test_collect_all.py create mode 100644 tests/test_handlers_github.py create mode 100644 tests/test_metrics_behavior.py diff --git a/.coverage b/.coverage deleted file mode 100644 index ac170622d430d895624681c89b1911e8c06d184a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI)O>Y}T7zglOukE$-;w_ZQilUOaKoT`c>r`q*dw{gfp`uV;3IY+~uGiyaiM{LY zuG2&nLT-^NRYHP0;#*LTd;t*R!nGU#l>?P3fe>a702T;900I#Be+zWq9Mj7a6YABE1GCnY zfoEFMi=M|%zqPz{VOd;QI(2SYMEk_dh@fSDUMvaEy)4?&6Aim5h3(XB%M5I1RRn7? zN!{^foo{rsk8X9Ga3PH6wW?jGs6eh#7H!XNnckLIms@jTg7|EK+zg@}l!>%g9Zn%; z;_zAF$%gc#V@W?M#f)8_9htrM$%tOwzhAvB!#bIsML(khwV@ros+?_5MwaWJFs0x$W3W=0$Ja+f^&?PEQHrz8_Me< zSLk9C8K0|9rx(0SWSl$>ZcyPIH(~7s@~J5}EVI*T)rkC~POHrqha#dA2kx-QVX=W5 zS2p#v+2vaH>N{{b6LP+yQB=pVFLmVh#h(+s$hoi@jyor~64U}-7|oYo+OI0qD&?ks zsY&m*s%9r}!|f_v{Az`s#y)+fpqHnn)a#eRhQd76njQGtE5(6Ev%Pyr)8XU8+YR@U z!A8QpsM!>5BkVF9EuxDs-=r|U46`$*1xf{|mSLH%{G#VBjcSEA4Qz_=l%TvRe&UO>1R<>71x)XW5JUOXe zEr!h~zFP5%KG1}cE4j-snjC6L)9-l6K=QCbGKuJol1U$lk5F~y^?Z40a#(fZbk$0{ zQt0Ygy?o$++KqYyk4yZ>4OD{n^t!+UH-hxVZ=1oI*jaYoO$DFM<;y1z3@bPnp<3bR z{Iw499=6&4q{g3)Jb7{zWUTmg&^v`xW1Yqg-k1=g!IJ zi3Sl06w}lxed?kQtD*bV%)X;?5Y2GgY5%>Mb)V zcD$JxXe19D?6sHkG~m*^o&I*hc6}VY8J&R#x9AGiB84PX{9b2dmM8eCaZjNiED(SI z1Rwwb2tWV=5P$##AOHaf>^%WZ&8Rxh|1-wVigAaYus{F;5P$##AOHafKmY;|fB*y_ z@Kg$nWwe7v@)sUQ_Nm(BMEo}Z#}_I`7LJWjtun@K#kg(!^;A|6g@ynGAOHafKmY;| zfB*y_009Ue3D@5P$## zAOHafKmY;|fB*y_0D++bm4c?&r%$ZB*e=8}m!2Ut3=BJ!{3ZTGDFT zD?a^)8~O(~^o9LJT~pT6gm|*;dV%yPW~Z_Et6#5e!}2tWV=5P$##AOHafKmY;|P?SZ@R@2O0 zd-MOoQEfd{NZ9NP<2&a6xg(lPl|wTB&mPzOQ~}8;UX+PyN{F+g7 zYPMJ`sr>nW&bXx*_l!S{KWP5{+ml^Elnw$AfB*y_009U<00Izz00bZafgJ*Qn(V9D zT=Vb$ z1w~aL009U<00Izz00bZa0SG_<0=o$C_x~~f-vtJ{AOHafKmY;|fB*y_009U<00MhR F;6HV~bx;5R diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d1e64b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +.coverage +*.coverage diff --git a/requirments.txt b/requirements.txt similarity index 100% rename from requirments.txt rename to requirements.txt diff --git a/run b/run index 6cacf51..b7c22e4 100644 --- a/run +++ b/run @@ -30,6 +30,10 @@ def do_test(): return 0 if r.returncode == 0 else 1 +def do_score(url_file): + from acmecli.cli import main + return main(["score", url_file]) + def main(): if len(sys.argv) < 2: print("Usage: run install|test|score ") diff --git a/src/acmecli.egg-info/SOURCES.txt b/src/acmecli.egg-info/SOURCES.txt index 7ab9ca9..989590b 100644 --- a/src/acmecli.egg-info/SOURCES.txt +++ b/src/acmecli.egg-info/SOURCES.txt @@ -1,5 +1,6 @@ pyproject.toml src/acmecli/__init__.py +src/acmecli/cache.py src/acmecli/cli.py src/acmecli/reporter.py src/acmecli/scoring.py @@ -8,8 +9,16 @@ src/acmecli.egg-info/PKG-INFO src/acmecli.egg-info/SOURCES.txt src/acmecli.egg-info/dependency_links.txt src/acmecli.egg-info/top_level.txt +src/acmecli/handlers/__init__.py +src/acmecli/handlers/github.py src/acmecli/metrics/__init__.py src/acmecli/metrics/base.py +src/acmecli/metrics/heuristic_metrics.py src/acmecli/metrics/license_metric.py tests/test_metrics_contract.py -tests/test_reporter_schema.py \ No newline at end of file +tests/test_reporter_schema.py +requirements.txt +tests/test_cli_flow.py +tests/test_collect_all.py +tests/test_handlers_github.py +tests/test_metrics_behavior.py diff --git a/src/acmecli/__pycache__/__init__.cpython-313.pyc b/src/acmecli/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index bc5983a657dd2780c0f5094f9c79236dd96f5550..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmey&%ge<81VR~yGeGoX5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa~ipvsFxJacWU< zOi_Mvc3O-}YMHL1v#)DRaZz$iVsdV3a!zJUe0*kJW=VX!UP0w84x8Nkl+v73yCPPg TevtXaAjU^#Mn=XWW*`dydQKqj diff --git a/src/acmecli/__pycache__/types.cpython-313.pyc b/src/acmecli/__pycache__/types.cpython-313.pyc deleted file mode 100644 index 9e236bbae12b3743bb76f6f42860460e687967ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4449 zcmbUkOKcm*b(cHbU4BGK)VFfv%9dS&KrFb3Ttn{Hr4>txOebDCagBAh>*a`|Lvq>K zr4t!F2oPYqH}zmZx%p_{+@5+gkV}pfXvCYjBt${Fy<@&r&nSHBDd7`+|prwl- zz_rB+Qh|G=;aA)`U<4UCg~u;nxj4JXwTl;u7mV2=CwUkHUA;74oV)g0o|?OM>C$}R z(z_S(vpl|dC4XVDn3K7>#N4~I?nFiuf6>vTovSc>D47zI4T;4JnJFhFlbCTMb_zJA z3Ya1=q!EE>fW;9rRHmB=Q#cT4nxN?{Sypoio-Ep|Onsw4oh=Q7xzc1+u6k~hIh2#S zU87ugY@e20wo>3aqjxGEocD>8OL5(@P{6Wy(z0r9saZulZCQV4+SQ2DYgtPb=K0l1 zoz`7oE1u6dQ0;o%^=-7B2UOJvchvLIUhGGp4NIU(Lzd;Cq+{8>&nmZ?K7}c440t>6 zlla^D^X4^=GS6hL^U0EVk={9dA%A(+^q6DXPK`R%is^&zys5?t8-OVS;Ca210PaiQ zCemwjK{~ru2$DT(^Fd~~O@@P`$5-bbEU$Zis(nR{vlE?p|Cfu`yno%i3e0f{FFoUNUwM$@gpMK`^bhYBpx<`c@c-nF6p?zHD@i^EF z{t_kk(|XmlOP(-TA(voB@E~s?7)Njv!H*G4Ai#3Ml?tphi9@thD98@4dmH)n&;C5sCKEv_v$hcQX73fkC2;n;5bJRtTwguhj?*CX zgF+oGB|_bZgQe7*##5ImoZtD-t~Tk`Fs#Ey0Py%7B=BSvLaXkqShXK$3Lakq!})9q zS31lXKSc%t+{^0+0Ffl(C)KyM81xO?yAqxoWez{>KY{aRr!{prw@B>WSP?Xl*n*~` zodqo+Ftk*%OG|ZW=`O8DU{ECXL}sST?d{I!>(ctWv}~6)(4`G_X+ww^!%%LZ;_}2* z+HjeF)%|o!1R(l{jRN3_I`u8jaT(?PUDS$P#?#EMHLPaC@+(k~2KSD-GRa#_&swq_ z-(@_zi+B0i24zbwtJ!sjT2SsQHII+IO6$sNQF8I#PL_4FNqUvLG*ULZi+5)ZCrCL_Vvc$XtYg6gTBFaW8>udjm^FZtQw*0TgTeuSa9N0o16-U#H!Y@zi9yDs+k`-My2=t8!~NU%&f33p*nc!AA<%d3=aVHIOT0JzGi-(+9SQ@VR z*DJ!_hZ&J7>;uG$xXxZpl1tQgmaQ7~Z9ckRhh5Ika8|T?KHObz!1Ya8-E;8ho$nP> z?9o8k)qq`x2_81{9jf8d3OCv+A4n3`rXH^;>Vi>YjgJZFGzF* zT-AYJ_v?l|`tZ=OhYRzx2D%U3pw? z9TLV4?#9XpmH~)$A%LQsQlLh6b_Vq49f?)x%XfCeLF5SE)Y*-QLN4?Dg<}Hdgl`Oy`@-|s12KjG zUC-_yI4n$R$UuM=_ud8o-L+^a$<2+07P0Gb()_onY-`~3W@>6x4-kE4 zGc~=c{#{P4pV*W~{u7JKliL!29YuTwge!O z1?;=zE_X;ix-9{SxPZN!QsgCheeNYpT9RKTWAYXG0gUkfrLM^5A6)-8u+9s6{uheA B_n!a& diff --git a/src/acmecli/cache.py b/src/acmecli/cache.py new file mode 100644 index 0000000..eb6e646 --- /dev/null +++ b/src/acmecli/cache.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Optional + + +class MemoryCache: + """In-memory cache that satisfies the Cache protocol used by metrics.""" + + def __init__(self) -> None: + self._store: dict[str, tuple[bytes, Optional[str]]] = {} + + def get(self, key: str) -> bytes | None: + entry = self._store.get(key) + if entry is None: + return None + return entry[0] + + def set(self, key: str, data: bytes, etag: str | None = None) -> None: + self._store[key] = (data, etag) diff --git a/src/acmecli/cli.py b/src/acmecli/cli.py index 6c5d6d3..8ee14fd 100644 --- a/src/acmecli/cli.py +++ b/src/acmecli/cli.py @@ -1,42 +1,151 @@ -import sys +from __future__ import annotations + +import logging +import os +import time from pathlib import Path -from .types import ReportRow +from typing import Dict + +from .handlers import GitHubHandler +from .metrics import heuristic_metrics # noqa: F401 - ensure registration +from .metrics import license_metric # noqa: F401 - ensure registration +from .metrics.base import MetricRunResult, collect_all from .reporter import write_ndjson +from .scoring import ScoringEngine +from .types import ReportRow, TargetSpec + +_LOGGER = logging.getLogger("acmecli") + def _classify(url: str) -> str: u = url.strip().lower() if "huggingface.co/datasets/" in u: return "DATASET" if "github.com/" in u: - return "CODE" + return "MODEL" if "huggingface.co/" in u: return "MODEL" return "CODE" -def _stub_row(name: str) -> ReportRow: - zero = 0.0 - return ReportRow( - name=name, category="MODEL", - net_score=zero, net_score_latency=0, - ramp_up_time=zero, ramp_up_time_latency=0, - bus_factor=zero, bus_factor_latency=0, - performance_claims=zero, performance_claims_latency=0, - license=zero, license_latency=0, - size_score={"raspberry_pi":0.0, "jetson_nano":0.0, "desktop_pc":0.0, "aws_server":0.0}, - size_score_latency=0, - dataset_and_code_score=zero, dataset_and_code_score_latency=0, - dataset_quality=zero, dataset_quality_latency=0, - code_quality=zero, code_quality_latency=0 + +def _setup_logging() -> None: + level = os.getenv("LOG_LEVEL", "0") + path = os.getenv("LOG_FILE") + + try: + numeric_level = int(level) + except ValueError: + numeric_level = 0 + + if numeric_level <= 0 or not path: + logging.basicConfig(level=logging.CRITICAL) + return + + log_level = logging.INFO if numeric_level == 1 else logging.DEBUG + + handler = logging.FileHandler(path, encoding="utf-8") + handler.setLevel(log_level) + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + + _LOGGER.setLevel(log_level) + _LOGGER.handlers.clear() + _LOGGER.addHandler(handler) + + +def _build_spec(url: str) -> TargetSpec: + category = _classify(url) + name = url.rstrip("/").split("/")[-1] or url + source = "GITHUB" if "github.com" in url.lower() else "HUGGINGFACE" + return TargetSpec(url=url, source=source, name=name, category=category) + + +def _clamp(value: float) -> float: + return max(0.0, min(1.0, value)) + + +def _value_for(metrics: Dict[str, float], name: str) -> float: + return _clamp(metrics.get(name, 0.0)) + + +def _latency_for(latencies: Dict[str, int], name: str) -> int: + return int(latencies.get(name, 0)) + + +def _metrics_to_maps(result: MetricRunResult) -> tuple[Dict[str, float], Dict[str, int]]: + value_map: Dict[str, float] = {} + latency_map: Dict[str, int] = {} + for metric in result.values: + value_map[metric.name] = metric.value + latency_map[metric.name] = metric.latency_ms + return value_map, latency_map + + +def _size_average(breakdown: Dict[str, float]) -> float: + if not breakdown: + return 0.0 + return sum(breakdown.values()) / len(breakdown) + + +def _process_url(url: str, handler: GitHubHandler, engine: ScoringEngine) -> None: + spec = _build_spec(url) + if spec.source != "GITHUB": + _LOGGER.info("Skipping unsupported source for URL: %s", url) + return + + result = collect_all(spec, handler) + values, latencies = _metrics_to_maps(result) + + size_breakdown = result.size_breakdown + size_average = _size_average(size_breakdown) + + compute_start = time.perf_counter() + net_score, metric_latency_sum = engine.compute(result.values, size_avg=size_average) + net_latency = metric_latency_sum + int((time.perf_counter() - compute_start) * 1000) + + repo_name = result.meta.get("full_name") or spec.name + + row = ReportRow( + name=repo_name, + category=spec.category, + net_score=_clamp(net_score), + net_score_latency=net_latency, + ramp_up_time=_value_for(values, "ramp_up_time"), + ramp_up_time_latency=_latency_for(latencies, "ramp_up_time"), + bus_factor=_value_for(values, "bus_factor"), + bus_factor_latency=_latency_for(latencies, "bus_factor"), + performance_claims=_value_for(values, "performance_claims"), + performance_claims_latency=_latency_for(latencies, "performance_claims"), + license=_value_for(values, "license"), + license_latency=_latency_for(latencies, "license"), + size_score=size_breakdown, + size_score_latency=_latency_for(latencies, "size"), + dataset_and_code_score=_value_for(values, "dataset_and_code_score"), + dataset_and_code_score_latency=_latency_for(latencies, "dataset_and_code_score"), + dataset_quality=_value_for(values, "dataset_quality"), + dataset_quality_latency=_latency_for(latencies, "dataset_quality"), + code_quality=_value_for(values, "code_quality"), + code_quality_latency=_latency_for(latencies, "code_quality"), ) + write_ndjson(row) + + def main(argv: list[str]) -> int: - # argv pattern: ["score", "/abs/path/URL_FILE"] + if len(argv) < 2: + print("Usage: acmecli score ") + return 1 + + _setup_logging() + handler = GitHubHandler(logger=_LOGGER) + engine = ScoringEngine() + _, url_file = argv lines = Path(url_file).read_text(encoding="utf-8").splitlines() + for raw in lines: url = raw.strip() if not url: continue - if _classify(url) == "MODEL": - write_ndjson(_stub_row(url)) + _process_url(url, handler, engine) + return 0 diff --git a/src/acmecli/handlers/__init__.py b/src/acmecli/handlers/__init__.py new file mode 100644 index 0000000..bc9eb89 --- /dev/null +++ b/src/acmecli/handlers/__init__.py @@ -0,0 +1,3 @@ +from .github import GitHubHandler + +__all__ = ["GitHubHandler"] diff --git a/src/acmecli/handlers/github.py b/src/acmecli/handlers/github.py new file mode 100644 index 0000000..1721124 --- /dev/null +++ b/src/acmecli/handlers/github.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import json +import logging +import os +import re +from typing import Iterable +from urllib import error, request + +from ..types import TargetSpec + +_GITHUB_API = "https://api.github.com" +_RAW_BASE = "https://raw.githubusercontent.com" + + +class GitHubHandler: + """Fetches lightweight repository metadata from GitHub's public API.""" + + def __init__(self, token: str | None = None, *, logger: logging.Logger | None = None) -> None: + self._token = token or os.getenv("GITHUB_TOKEN") or os.getenv("GITHUB_TOKEN_ACME") + self._log = logger or logging.getLogger("acmecli.github") + + # --- SourceHandler protocol ------------------------------------------------- + def resolve_revision(self, url: str) -> str: + owner, repo = _split_repo(url) + data = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}") + if not data: + return "main" + return data.get("default_branch") or "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + owner, repo = _split_repo(spec.url) + repo_url = f"{_GITHUB_API}/repos/{owner}/{repo}" + repo_data = self._fetch_json(repo_url) + if not repo_data: + return { + "owner": owner, + "repo": repo, + "full_name": f"{owner}/{repo}", + "stars": 0, + "forks": 0, + "watchers": 0, + "open_issues": 0, + "default_branch": "main", + "license": "", + "size_kb": 0, + "readme_text": "", + "contributors_count": 0, + "recent_commits": 0, + "topics": [], + "description": "", + "has_wiki": False, + "pushed_at": None, + } + + default_branch = repo_data.get("default_branch") or "main" + readme_text = self._fetch_readme(owner, repo, default_branch) + contributors_count = self._fetch_contributors_count(owner, repo) + recent_commits = self._fetch_recent_commits(owner, repo) + + return { + "owner": owner, + "repo": repo, + "full_name": repo_data.get("full_name") or f"{owner}/{repo}", + "stars": repo_data.get("stargazers_count", 0), + "forks": repo_data.get("forks_count", 0), + "watchers": repo_data.get("subscribers_count", 0), + "open_issues": repo_data.get("open_issues_count", 0), + "default_branch": default_branch, + "license": (repo_data.get("license") or {}).get("name") or "", + "size_kb": repo_data.get("size", 0), + "readme_text": readme_text, + "contributors_count": contributors_count, + "recent_commits": recent_commits, + "topics": repo_data.get("topics", []), + "description": repo_data.get("description") or "", + "has_wiki": bool(repo_data.get("has_wiki", False)), + "pushed_at": repo_data.get("pushed_at"), + "created_at": repo_data.get("created_at"), + "updated_at": repo_data.get("updated_at"), + } + + def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: + meta = self.fetch_meta(spec) + if meta.get("readme_text"): + yield "README.md", meta["readme_text"].encode("utf-8", errors="ignore") + + # --- Internal helpers ------------------------------------------------------- + def _fetch_json(self, url: str) -> dict: + try: + req = request.Request(url, headers=self._headers()) + with request.urlopen(req, timeout=15) as resp: + payload = resp.read() + return json.loads(payload.decode("utf-8")) + except error.HTTPError as exc: + if exc.code == 403: + self._log.debug("GitHub API rate limit hit for %s", url) + else: + self._log.debug("GitHub API error %s for %s", exc.code, url) + except Exception as exc: # pragma: no cover - defensive + self._log.debug("GitHub API fetch failed for %s: %s", url, exc) + return {} + + def _fetch_text(self, url: str) -> str: + try: + req = request.Request(url, headers=self._headers(accept="text/plain")) + with request.urlopen(req, timeout=15) as resp: + payload = resp.read() + return payload.decode("utf-8", errors="ignore") + except Exception: + return "" + + def _fetch_readme(self, owner: str, repo: str, branch: str) -> str: + return self._fetch_text(f"{_RAW_BASE}/{owner}/{repo}/{branch}/README.md") + + def _fetch_contributors_count(self, owner: str, repo: str) -> int: + data = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}/contributors?per_page=100") + if isinstance(data, list): + return len(data) + return 0 + + def _fetch_recent_commits(self, owner: str, repo: str) -> int: + commits = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}/commits?per_page=30") + if isinstance(commits, list): + return len(commits) + return 0 + + def _headers(self, *, accept: str = "application/vnd.github+json") -> dict[str, str]: + headers = { + "Accept": accept, + "User-Agent": "acmecli/0.0.1", + } + if self._token: + headers["Authorization"] = f"Bearer {self._token}" + return headers + + +def _split_repo(url: str) -> tuple[str, str]: + match = re.search(r"github\.com/([^/]+)/([^/#?]+)", url) + if not match: + raise ValueError(f"Unsupported GitHub URL: {url}") + owner, repo = match.group(1), match.group(2) + if repo.endswith(".git"): + repo = repo[:-4] + return owner, repo diff --git a/src/acmecli/metrics/__init__.py b/src/acmecli/metrics/__init__.py index e69de29..3561957 100644 --- a/src/acmecli/metrics/__init__.py +++ b/src/acmecli/metrics/__init__.py @@ -0,0 +1,21 @@ +from .license_metric import LicenseMetric # noqa: F401 +from .heuristic_metrics import ( # noqa: F401 + BusFactorMetric, + CodeQualityMetric, + DatasetAvailabilityMetric, + DatasetQualityMetric, + PerformanceClaimsMetric, + RampUpMetric, + SizeMetric, +) + +__all__ = [ + "LicenseMetric", + "RampUpMetric", + "BusFactorMetric", + "PerformanceClaimsMetric", + "SizeMetric", + "DatasetAvailabilityMetric", + "DatasetQualityMetric", + "CodeQualityMetric", +] diff --git a/src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc b/src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 63ad76f244e30b9126f7c00fd73e9926221ec436..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 152 zcmey&%ge<81d|dEXMpI(AOZ#$p^VQgK*m&tbOudEzm*I{OhDdekkl=6XRDad;?$zz zn4sYP3?Ot8v;(kI2nmG(xzbjO(@F?Sk-3=bL2`F`zErBZ zl!>x+>CVdE!NiQZSO}1qxIMbeS z&fpd>U}O^_f}{nk!ko=Y+nmC%8QB>uJ*QMiD|6sbKVofvDCGmz(kg;kg;yCe2lStF0#Q; zqITON-j_P)ZanL-UiYQ;M)L+9%5sd^c1-M;g*k@pnXVc*gI9|%K-Wh9vt4+wsBJ-F z7TpdjXg^GQq9|lG2np7WR5q!l@3q(Yu9B!Y=JC5X{E0lc`?$Te!4<|_#3LCGQ$F8K z@z+Ah)#6YGC=>H|iFm~{@ocxt29Du7z4T${{m$vrbK0Dm2!wU*m4`)|C1_XGt`Sn9 z{bjUu=vxRZR4vRFUF!EmmgS`=(>zoC{%p!;z-61UWIzWGNBLkpl#lSLk@h8O&A`MV pg#3V&pU^pn&aZ3DjgHF4Eg?15(4G^|) z*4%apOAVNXc@Tvpng_2u4@)@!OGfTApldYUb3Uw}b9A$y3-@G-QK*X| z>Lx*~boqk-yD}BKGSpSkA7Cio4U+9xqIOn-a7XG&p7LQ359B$QZW`J}*^N{xG34qK zamUVRm7GKH01W|X3Ii=+qAe`X5w=$lj#m@~&s}syu?`>GSQ4)15>j5YMJYonqKs4B zN^?qA--|+-D4CNucc`*1#yO^K#k7k%6T)FexPUSpU=mKo&8(QWMZZyj)FIr01!`mc z2WZ+xpK$bR1%ek{qU2Hw<;gD-d?%)j;Ap6g0LsUIP!Yg zP?b04<>%*{yL-AA_TyNFsjgk<#Eu{K2MKMEwxT4(5=Ck|33{^W;LCKtYmAubE$uwy z&C}&LNM#c4`8}n$8@`Ldi_cD`bo8MU_QNc z<-j?s&3?UbTEFq6e&eLxIyN8IUq9Y{eD%&r?Tv%-*>wGA@tftZmcNxJ)3+bFx3Nxj zutsXVS=FxZGhN@;W#8}hw+Au%Q@;P{Ac)6brhUH?p-ST@kxAe8vB4F11(64dJVE4U zaF!Jh0T)Q=n(wPLNTbjXQhKis2B}n(&i|dW;J&dsQCvdZCG{CRt-Q3qI None: REGISTRY.append(metric) + + +def collect_all( + spec: TargetSpec, + handler: SourceHandler, + cache: Cache | None = None, + max_workers: int | None = None, +) -> MetricRunResult: + """Collect and score all registered metrics in parallel.""" + + if cache is None: + cache = MemoryCache() + + meta = handler.fetch_meta(spec) + cache.set("meta", json.dumps(meta).encode("utf-8")) + + if not REGISTRY: + empty_breakdown = {"raspberry_pi": 0.0, "jetson_nano": 0.0, "desktop_pc": 0.0, "aws_server": 0.0} + return MetricRunResult([], empty_breakdown, 0, meta) + + max_workers = max_workers or min(8, max(1, len(REGISTRY))) + values: list[MetricValue] = [] + size_breakdown: dict[str, float] = {"raspberry_pi": 0.0, "jetson_nano": 0.0, "desktop_pc": 0.0, "aws_server": 0.0} + + def _execute(metric: Metric) -> tuple[MetricValue, dict]: + signals = metric.collect(spec, handler, cache) + value = metric.score(signals) + return value, signals # type: ignore[return-value] + + with ThreadPoolExecutor(max_workers=max_workers) as pool: + future_map = {pool.submit(_execute, metric): metric for metric in REGISTRY} + for future in as_completed(future_map): + value, signals = future.result() + values.append(value) + if value.name == "size": + size_breakdown = signals.get("size_breakdown", size_breakdown) + + total_latency = sum(v.latency_ms for v in values) + return MetricRunResult(values=values, size_breakdown=size_breakdown, total_latency_ms=total_latency, meta=meta) diff --git a/src/acmecli/metrics/heuristic_metrics.py b/src/acmecli/metrics/heuristic_metrics.py new file mode 100644 index 0000000..b0bdc83 --- /dev/null +++ b/src/acmecli/metrics/heuristic_metrics.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import json +import math +import re +import time + +from ..types import Cache, MetricValue, Signals, SourceHandler, TargetSpec +from .base import register + +_KEYWORDS_DATASET = ["dataset", "data set", "training data", "hugging face", "download"] +_KEYWORDS_CODE = ["script", "example", "usage", "cli", "run", "train"] +_KEYWORDS_PERF = ["accuracy", "f1", "precision", "recall", "benchmark", "latency", "throughput", "performance"] + + +def _load_meta(cache: Cache) -> dict: + raw = cache.get("meta") + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + +def _count_keywords(text: str, keywords: list[str]) -> int: + text_lower = text.lower() + return sum(text_lower.count(keyword) for keyword in keywords) + + +def _readme_sections_score(text: str) -> float: + lowered = text.lower() + score = 0.0 + if "installation" in lowered or "getting started" in lowered: + score += 0.3 + if "usage" in lowered or "example" in lowered: + score += 0.3 + if "contributing" in lowered or "license" in lowered: + score += 0.2 + return min(0.8, score) + + +def _device_score(size_mb: float, threshold_mb: float) -> float: + if size_mb <= threshold_mb: + return 1.0 + if size_mb >= threshold_mb * 2: + return 0.0 + return max(0.0, 1.0 - (size_mb - threshold_mb) / threshold_mb) + + +class RampUpMetric: + name = "ramp_up_time" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + "stars": meta.get("stars", 0), + "topics": meta.get("topics", []), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + word_count = len(re.findall(r"\w+", readme)) + base = min(0.6, word_count / 1500) + sections = _readme_sections_score(readme) + topic_bonus = 0.1 if signals.get("topics") else 0.0 + total = min(1.0, base + sections + topic_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class BusFactorMetric: + name = "bus_factor" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "contributors": {"count": meta.get("contributors_count", 0)}, + "recent_commits": meta.get("recent_commits", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + contributors = signals.get("contributors", {}).get("count", 0) + recent_commits = signals.get("recent_commits", 0) + contrib_score = min(1.0, contributors / 5) + freshness = min(0.4, recent_commits / 50) + total = min(1.0, contrib_score + freshness) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class PerformanceClaimsMetric: + name = "performance_claims" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + "stars": meta.get("stars", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + perf_mentions = _count_keywords(readme, _KEYWORDS_PERF) + base = min(0.7, perf_mentions / 5) + stars = signals.get("stars", 0) + stars_bonus = min(0.3, math.log10(stars + 1) / 10) + total = min(1.0, base + stars_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class SizeMetric: + name = "size" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + size_kb = meta.get("size_kb", 0) + size_mb = size_kb / 1024 + breakdown = { + "raspberry_pi": _device_score(size_mb, 50), + "jetson_nano": _device_score(size_mb, 200), + "desktop_pc": _device_score(size_mb, 1024), + "aws_server": _device_score(size_mb, 4096), + } + return { + "size_breakdown": breakdown, + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + breakdown = signals.get("size_breakdown", {}) + if not breakdown: + value = 0.0 + else: + value = sum(breakdown.values()) / len(breakdown) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, max(0.0, min(1.0, value)), latency_ms) + + +class DatasetAvailabilityMetric: + name = "dataset_and_code_score" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + readme = meta.get("readme_text", "") + return { + "readme_text": readme, + "has_examples": any(keyword in readme.lower() for keyword in _KEYWORDS_CODE), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + dataset_mentions = _count_keywords(readme, _KEYWORDS_DATASET) + example_bonus = 0.3 if signals.get("has_examples") else 0.0 + base = min(0.7, dataset_mentions / 5) + total = min(1.0, base + example_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class DatasetQualityMetric: + name = "dataset_quality" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + mentions = _count_keywords(readme, ["benchmark", "evaluation", "split", "validation", "license"]) + total = min(1.0, mentions / 5) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class CodeQualityMetric: + name = "code_quality" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "open_issues": meta.get("open_issues", 0), + "recent_commits": meta.get("recent_commits", 0), + "forks": meta.get("forks", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + open_issues = signals.get("open_issues", 0) + recent_commits = signals.get("recent_commits", 0) + forks = signals.get("forks", 0) + + activity = min(0.6, recent_commits / 50) + balance = 0.4 if open_issues == 0 else max(0.0, 0.4 - (open_issues / 200)) + fork_bonus = min(0.2, forks / 200) + total = max(0.0, min(1.0, activity + balance + fork_bonus)) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +register(RampUpMetric()) +register(BusFactorMetric()) +register(PerformanceClaimsMetric()) +register(SizeMetric()) +register(DatasetAvailabilityMetric()) +register(DatasetQualityMetric()) +register(CodeQualityMetric()) diff --git a/src/acmecli/metrics/license_metric.py b/src/acmecli/metrics/license_metric.py index 838708a..41ac1c8 100644 --- a/src/acmecli/metrics/license_metric.py +++ b/src/acmecli/metrics/license_metric.py @@ -1,18 +1,51 @@ +from __future__ import annotations + +import json import time -from ..types import Metric, Signals, TargetSpec, SourceHandler, Cache, MetricValue + +from ..types import Cache, Metric, MetricValue, Signals, SourceHandler, TargetSpec from .base import register +_COMPATIBLE_KEYWORDS = ["lgpl", "mit", "bsd", "apache", "mpl", "cc-by"] +_RESTRICTIVE_KEYWORDS = ["gpl", "agpl"] + + +def _load_meta(cache: Cache) -> dict: + raw = cache.get("meta") + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + class LicenseMetric: name = "license" def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: - # Week 1 stub: no network; just return empty signals - return {} + meta = _load_meta(cache) + signals: Signals = { + "license_name": meta.get("license", ""), + "readme_text": meta.get("readme_text", ""), + } + return signals def score(self, signals: Signals) -> MetricValue: t0 = time.perf_counter() - value = 0.0 # TODO: map license presence/compatibility → [0,1] + license_name = (signals.get("license_name") or "").lower() + readme_text = (signals.get("readme_text") or "").lower() + + value = 0.0 + if license_name: + if any(keyword in license_name for keyword in _COMPATIBLE_KEYWORDS): + value = 1.0 + elif any(keyword in license_name for keyword in _RESTRICTIVE_KEYWORDS): + value = 0.3 + else: + value = 0.6 + elif "license" in readme_text: + value = 0.4 + latency_ms = int((time.perf_counter() - t0) * 1000) - return MetricValue(self.name, value, latency_ms) + return MetricValue(self.name, max(0.0, min(1.0, value)), latency_ms) + register(LicenseMetric()) diff --git a/src/acmecli/types.py b/src/acmecli/types.py index 32ec282..8ee207e 100644 --- a/src/acmecli/types.py +++ b/src/acmecli/types.py @@ -1,8 +1,11 @@ -from dataclasses import dataclass, asdict -from typing import Protocol, Iterable, TypedDict, Literal, Optional, Dict +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Iterable, Literal, NotRequired, Optional, Protocol, TypedDict Category = Literal["MODEL", "DATASET", "CODE"] -Source = Literal["GITHUB", "HUGGINGFACE", "LOCAL"] +Source = Literal["GITHUB", "HUGGINGFACE", "LOCAL"] + @dataclass(frozen=True) class TargetSpec: @@ -12,19 +15,29 @@ class TargetSpec: category: Category revision: Optional[str] = None # commit/tag if known + class Signals(TypedDict, total=False): readme_text: str license_name: str contributors: Dict[str, int] stars: int downloads: int + forks: int + open_issues: int + recent_commits: int + size_breakdown: Dict[str, float] + topics: list[str] + has_examples: bool + documentation_score: float + @dataclass(frozen=True) class MetricValue: - name: str # e.g. "ramp_up_time" - value: float # [0,1] + name: str # e.g. "ramp_up_time" + value: float # [0,1] latency_ms: int + @dataclass(frozen=True) class ReportRow: # Required NDJSON fields + per-metric latencies (values are stubs for now) @@ -40,7 +53,7 @@ class ReportRow: performance_claims_latency: int license: float license_latency: int - size_score: Dict[str, float] # {raspberry_pi, jetson_nano, desktop_pc, aws_server} + size_score: Dict[str, float] # {raspberry_pi, jetson_nano, desktop_pc, aws_server} size_score_latency: int dataset_and_code_score: float dataset_and_code_score_latency: int @@ -49,16 +62,31 @@ class ReportRow: code_quality: float code_quality_latency: int + class SourceHandler(Protocol): - def resolve_revision(self, url: str) -> str: ... - def fetch_meta(self, spec: TargetSpec) -> dict: ... - def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: ... + def resolve_revision(self, url: str) -> str: + ... + + def fetch_meta(self, spec: TargetSpec) -> dict: + ... + + def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: + ... + class Cache(Protocol): - def get(self, key: str) -> bytes | None: ... - def set(self, key: str, data: bytes, etag: str | None = None) -> None: ... + def get(self, key: str) -> bytes | None: + ... + + def set(self, key: str, data: bytes, etag: str | None = None) -> None: + ... + class Metric(Protocol): name: str - def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: ... - def score(self, signals: Signals) -> MetricValue: ... + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + ... + + def score(self, signals: Signals) -> MetricValue: + ... diff --git a/tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc b/tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc deleted file mode 100644 index ba1c99c67a5846d80fc96190543bdd1b9ee152ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3326 zcmdTGU2hvjaPPz4i9?%~^n-wt3Mq%El@B`!Ar7Ra5K>wsAC?M^bh=(Hu8VzVcAY?i zDn*EgmLj!0R75->-de#Q;FUj+kcO5M2nnA0R^cc5#LVs8`D`L09vRuQGqW?hvom`$ z+v@F25`f&*AFjM75b_5yK8POCIs?QVq7qd&PpIII;-bXF3|W+EB!e^J`RHPd##~xD zA74z+1kmycLX&C)aH=9G(MP!8MMZGp#dB}GslHQu7wF!%s)lJX^Mcu=Rm1T1bcZNl z3I46mz%_6{I6=z6d_T#1v_;COV0pYK3T!HlUl#mqIRD>yMMNmX1R^#^p?}=>bbrWP_cGMB-4C9^2IFn~* zmP*W|Y>C>&`zuSYnQN!#&t5pU)HGR>xmc^aw~1+nZ8fQGG_#F$Cox+wE%T!WopUBo zRikF5%9?atyP`AA+t#zp&em*0ud%soU=)Gf*#MwNWRq;C^Y<3M{@~WyjkT@gKlKgY zTlkCN_4?%Gps?|x&cMB9=J}DvdQM@R~Bv1wgzY7mo}Zs|>!0Y_4}8$ZEVn3>=Vx5+bfFO$j8 z6ndCBD4z5gP{RPR3ZPB$b2|Us!frahE$2UvzC+NIAI1?OIYIMbzkm(cl}m1XZwyig=bTu*XZ%%nR_H z$*O3pr3ljYa!?augp{H_x4aD%BB(+|^s5Voi7H)sJtwO2rBkIyPRe^bhOG9jO6@OK zjg*5LQpAhY1zi>IWQRxlSB0Ra_~+1{?y%b5qreI(T;xFiCBNbvs2TA!)o50BD_l&C zsqs>5RB*X?8A=d`&{{`)S$l+f7x&llM_%tNu^?D7{G(9&@+F}N=j{e7p(gCRrR2=F zkna^gGRxn0L}vdCCri1(PzUSCS7y=uCY^0L2WjC);zHxOtS?FN#q=}prz*0nlIUdzmJ-}BH=B#J$r;^&JS*LJW7(E`$^F!shIiH^lGt;aWbfG`Bfp@9_ zw^Q-wf##lDqqw3v&6AsF9;VXU^3azf zUyQ(Tdt@gy1ml+BB48fK?ca_(bbVsW;G+A;glB;U+3n##2I@YyhfDK`U0glT2@?-f zy|)TC3fuDF*1U^$QiBg<@AsoTc)bvCoY(`|cp%$pb2#BC@8W8KPC(q32k9ULM(J_l z{v*V_4$g6nzd*7)8}NL{Qu9MvZNjr75IHeV#_!h=-Kd*Jt(x^-XxYm;yp^!yI?r}b zbw0`5E9<@^++5uyu2(#(@!7h)yjnBoXaQEjuFmi-J(dJP_>GWH$sSRDCDVV(f-u>N J%EGg*?cY|;g#`cr diff --git a/tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc b/tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc deleted file mode 100644 index f2d1c7740ee1ccca076793052a6120b33f8f568f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2003 zcmah}TW=dh6rS;}@0Y}mb8To{he8(`<6NAeVgpqY38l1^vWkS1NUP0ylB`(oZfDj6 z2O;%=2Y5n4s>CyI`~_b50~DwbBP3Mu#M?sNcw%POvv#0@m3_{4zVpqQGZ)Q8v6utw z`|9-D(=QSL{KfW;P39g09q8~bz)<+au0j$uu&d%^jb{>fQ@d%LW>{GOICBCu z^>;q_z6NP_5BVMr_PjGo)Ek;K^9uW4wAn{aTYvr6&xiE4s>TQcp3P)-%jz z1|Z$KC9gFU%)*%I?E)K%~L ze&Y5!0C;ORhnem-034`NwwID<11rCAD>wLGe`u3N($l)afVmqwNTd(;ESdK#_d`D!TV_+Mp0D)q&Nvds3yRrMG2 ztNUsbiYR$dmi>&?sz`3+*~*f#mh^~S!VYv{2l}fgK&#!Ss%!QTRlBI?(TrsV=)}VV zn!NX5d*@x6bx~jt%fskI)}`f`GaM#&tpVnTP7B!V`9|M20-F`Ajz>~Rb4Pt*9Gh0) zVOkmCIA8G*KK5|WbS-39j%oJ@UHCT_>(h>HA(tRJF$8fYN9=PHwN5J|9Oq}cW?&Mw zI@9eMme-ZqtIy}-aT7BCux~ncFrc&lAYx@fRF$&ocqBC%{43GvXz;vwSHL7FgOi>f26WyM(43k0}V9Gsvmece+AqoH*W* z=@{bTWDQ~-BVK`)?uf~4&)vb89q^PPjxsRNhrVOFX23F93NP|5@_d@8*_H8NeLPr; zf~d;Ic)y^GUnUWj_st$sgxQTaMLo0O z4`^NNLX=>eX`Gsb$sYD?jJn3LjhrsQ{Qc%zN8VyD#3S%aZt72v&!>JaPK65hS8oXa ztTG)YS^P|UMfg8bU}io{Glrq&N`x|?IvZv=mjSnyBa{P+*TOvKa-iLbPyv*yVUcqM z@Y2f>DuL=uIKjCRc=c9wHYnw0g*L>_#x~5s5B0KPr5k rn-y+e= sum(m.latency_ms for m in metric_values) + assert row.dataset_quality == 0.4 + assert row.performance_claims == 0.5 + + +def test_classify_variants(): + assert cli._classify('https://huggingface.co/datasets/my-dataset') == 'DATASET' + assert cli._classify('https://github.com/acme/project') == 'MODEL' + assert cli._classify('https://huggingface.co/acme/model') == 'MODEL' + assert cli._classify('https://example.com/something') == 'CODE' + + +def test_setup_logging_creates_file(tmp_path, monkeypatch): + log_file = tmp_path / 'acme.log' + monkeypatch.setenv('LOG_FILE', str(log_file)) + monkeypatch.setenv('LOG_LEVEL', '1') + old_handlers = list(cli._LOGGER.handlers) + cli._LOGGER.handlers.clear() + try: + cli._setup_logging() + assert cli._LOGGER.handlers, 'expected logger to have handlers' + cli._LOGGER.info('hello world') + for handler in list(cli._LOGGER.handlers): + handler.flush() + assert log_file.exists() + finally: + for handler in list(cli._LOGGER.handlers): + handler.close() + cli._LOGGER.removeHandler(handler) + for handler in old_handlers: + cli._LOGGER.addHandler(handler) + monkeypatch.delenv('LOG_FILE', raising=False) + monkeypatch.delenv('LOG_LEVEL', raising=False) + + +def test_build_spec_sets_fields(): + spec = cli._build_spec('https://github.com/acme/tools') + assert spec.source == 'GITHUB' + assert spec.category == 'MODEL' + assert spec.name == 'tools' diff --git a/tests/test_collect_all.py b/tests/test_collect_all.py new file mode 100644 index 0000000..50283a8 --- /dev/null +++ b/tests/test_collect_all.py @@ -0,0 +1,70 @@ +import json + +import pytest + +from acmecli.cache import MemoryCache +from acmecli.metrics.base import MetricRunResult, collect_all +from acmecli.types import MetricValue, TargetSpec + + +class StubHandler: + def __init__(self, meta): + self._meta = meta + + def resolve_revision(self, url: str) -> str: + return "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + return self._meta + + def stream_files(self, spec: TargetSpec, patterns): + return iter(()) + + +def _spec() -> TargetSpec: + return TargetSpec( + url="https://github.com/acme/test", + source="GITHUB", + name="test", + category="MODEL", + ) + + +@pytest.fixture() +def rich_meta(): + return { + "full_name": "acme/test", + "license": "MIT License", + "readme_text": "Installation usage benchmark dataset split validation example", + "stars": 120, + "topics": ["ml"], + "contributors_count": 6, + "recent_commits": 45, + "size_kb": 4096, + "open_issues": 2, + "forks": 40, + } + + +def test_collect_all_returns_metric_values(rich_meta): + handler = StubHandler(rich_meta) + result = collect_all(_spec(), handler, cache=MemoryCache()) + assert isinstance(result, MetricRunResult) + assert result.meta["full_name"] == "acme/test" + assert result.values, "expected registered metrics to produce values" + for metric in result.values: + assert 0.0 <= metric.value <= 1.0 + assert metric.latency_ms >= 0 + assert set(result.size_breakdown.keys()) == { + "raspberry_pi", + "jetson_nano", + "desktop_pc", + "aws_server", + } + + +def test_collect_all_latency_matches_sum(rich_meta): + handler = StubHandler(rich_meta) + result = collect_all(_spec(), handler, cache=MemoryCache()) + summed = sum(m.latency_ms for m in result.values) + assert result.total_latency_ms == summed diff --git a/tests/test_handlers_github.py b/tests/test_handlers_github.py new file mode 100644 index 0000000..949aee6 --- /dev/null +++ b/tests/test_handlers_github.py @@ -0,0 +1,81 @@ +import logging +from types import SimpleNamespace + +import pytest + +from acmecli.handlers.github import GitHubHandler, _split_repo +from acmecli.types import TargetSpec + + +def test_split_repo_removes_suffix(): + owner, repo = _split_repo("https://github.com/acme/project.git") + assert owner == "acme" + assert repo == "project" + + +def test_split_repo_invalid_url(): + with pytest.raises(ValueError): + _split_repo("https://example.com/notgithub") + + +def test_github_handler_fetch_meta(monkeypatch): + repo_data = { + "full_name": "octocat/hello-world", + "stargazers_count": 42, + "forks_count": 7, + "subscribers_count": 3, + "open_issues_count": 1, + "default_branch": "main", + "license": {"name": "MIT License"}, + "size": 2048, + "topics": ["ml"], + "description": "Sample repo", + "has_wiki": True, + "pushed_at": "2024-01-01T00:00:00Z", + "created_at": "2023-01-01T00:00:00Z", + "updated_at": "2024-01-01T00:00:00Z", + } + + def fake_fetch_json(url: str): + if url.endswith("contributors?per_page=100"): + return [{"login": "a"}, {"login": "b"}] + if url.endswith("commits?per_page=30"): + return [{"sha": "1"}, {"sha": "2"}, {"sha": "3"}] + return repo_data + + handler = GitHubHandler(logger=logging.getLogger("test")) + monkeypatch.setattr(handler, "_fetch_json", fake_fetch_json) + monkeypatch.setattr(handler, "_fetch_readme", lambda owner, repo, branch: "# Installation\nUsage") + + spec = TargetSpec( + url="https://github.com/octocat/hello-world", + source="GITHUB", + name="hello-world", + category="MODEL", + ) + + meta = handler.fetch_meta(spec) + assert meta["full_name"] == "octocat/hello-world" + assert meta["stars"] == 42 + assert meta["license"] == "MIT License" + assert meta["contributors_count"] == 2 + assert meta["recent_commits"] == 3 + assert meta["readme_text"].startswith("# Installation") + + +def test_github_handler_default_meta(monkeypatch): + handler = GitHubHandler(logger=logging.getLogger("test")) + monkeypatch.setattr(handler, "_fetch_json", lambda url: {}) + monkeypatch.setattr(handler, "_fetch_readme", lambda owner, repo, branch: "") + + spec = TargetSpec( + url="https://github.com/octocat/unknown", + source="GITHUB", + name="unknown", + category="MODEL", + ) + + meta = handler.fetch_meta(spec) + assert meta["stars"] == 0 + assert meta["license"] == "" + assert meta["full_name"] == "octocat/unknown" diff --git a/tests/test_metrics_behavior.py b/tests/test_metrics_behavior.py new file mode 100644 index 0000000..b892ef6 --- /dev/null +++ b/tests/test_metrics_behavior.py @@ -0,0 +1,154 @@ +import json +import math + +import pytest + +from acmecli.cache import MemoryCache +from acmecli.metrics.heuristic_metrics import ( + BusFactorMetric, + CodeQualityMetric, + DatasetAvailabilityMetric, + DatasetQualityMetric, + PerformanceClaimsMetric, + RampUpMetric, + SizeMetric, +) +from acmecli.metrics.license_metric import LicenseMetric +from acmecli.types import TargetSpec + + +class _DummyHandler: + def __init__(self, meta): + self._meta = meta + + def resolve_revision(self, url: str) -> str: + return "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + return self._meta + + def stream_files(self, spec: TargetSpec, patterns): + return iter(()) + + +def _cache_with_meta(meta: dict) -> MemoryCache: + cache = MemoryCache() + cache.set("meta", json.dumps(meta).encode("utf-8")) + return cache + + +def _spec() -> TargetSpec: + return TargetSpec( + url="https://github.com/example/repo", + source="GITHUB", + name="repo", + category="MODEL", + ) + + +@pytest.mark.parametrize( + "license_name,readme_text,expected", + [ + ("MIT License", "", 1.0), + ("GNU GPL v3", "", 0.3), + ("", "This project includes a License section", 0.4), + ], +) +def test_license_metric_scores(license_name, readme_text, expected): + metric = LicenseMetric() + signals = {"license_name": license_name, "readme_text": readme_text} + mv = metric.score(signals) + assert mv.name == "license" + assert mv.value == pytest.approx(expected, abs=0.05) + assert mv.latency_ms >= 0 + + +def test_ramp_up_metric_high_score(): + readme = "Installation usage contributing " + "word " * 1600 + meta = {"readme_text": readme, "stars": 50, "topics": ["ml"]} + metric = RampUpMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.9 + assert mv.latency_ms >= 0 + + +def test_ramp_up_metric_low_score(): + meta = {"readme_text": "tiny", "stars": 0, "topics": []} + metric = RampUpMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value < 0.3 + + +def test_bus_factor_metric_range(): + meta = {"contributors_count": 10, "recent_commits": 60} + metric = BusFactorMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert 0.9 <= mv.value <= 1.0 + + +def test_bus_factor_metric_low_activity(): + meta = {"contributors_count": 0, "recent_commits": 0} + metric = BusFactorMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == 0.0 + + +def test_performance_claims_metric_detects_keywords(): + readme = "accuracy accuracy benchmark throughput" + meta = {"readme_text": readme, "stars": 200} + metric = PerformanceClaimsMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.6 + + +def test_performance_claims_metric_without_keywords(): + meta = {"readme_text": "no claims here", "stars": 0} + metric = PerformanceClaimsMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == pytest.approx(0.0) + + +def test_size_metric_average(): + meta = {"size_kb": 1024 * 3} + metric = SizeMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert set(signals["size_breakdown"].keys()) == { + "raspberry_pi", + "jetson_nano", + "desktop_pc", + "aws_server", + } + assert mv.value == pytest.approx(1.0) + + +def test_dataset_availability_metric_examples(): + readme = "Training dataset download script example usage" + meta = {"readme_text": readme} + metric = DatasetAvailabilityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.6 + + +def test_dataset_quality_metric_mentions(): + readme = "Benchmark evaluation split validation license" + meta = {"readme_text": readme} + metric = DatasetQualityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == pytest.approx(1.0) + + +def test_code_quality_metric_balances_signals(): + meta = {"open_issues": 1, "recent_commits": 40, "forks": 80} + metric = CodeQualityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert 0.5 < mv.value <= 1.0 diff --git a/tests/test_reporter_schema.py b/tests/test_reporter_schema.py index ba0dbcd..4f45394 100644 --- a/tests/test_reporter_schema.py +++ b/tests/test_reporter_schema.py @@ -14,3 +14,35 @@ def test_reportrow_has_required_fields(): code_quality=0.0, code_quality_latency=0 ) assert row.category == "MODEL" + + +from acmecli.reporter import write_ndjson + + +def test_reporter_prints_ndjson(capsys): + row = ReportRow( + name='demo', + category='MODEL', + net_score=0.5, + net_score_latency=10, + ramp_up_time=0.4, + ramp_up_time_latency=5, + bus_factor=0.3, + bus_factor_latency=4, + performance_claims=0.2, + performance_claims_latency=3, + license=0.9, + license_latency=2, + size_score={'raspberry_pi': 0.5, 'jetson_nano': 0.6, 'desktop_pc': 0.7, 'aws_server': 0.8}, + size_score_latency=1, + dataset_and_code_score=0.6, + dataset_and_code_score_latency=2, + dataset_quality=0.7, + dataset_quality_latency=2, + code_quality=0.8, + code_quality_latency=2, + ) + write_ndjson(row) + captured = capsys.readouterr().out.strip() + assert '"name": "demo"' in captured + assert '"net_score": 0.5' in captured