diff --git a/.coverage b/.coverage deleted file mode 100644 index ac17062..0000000 Binary files a/.coverage and /dev/null differ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d1e64b --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +.coverage +*.coverage diff --git a/requirments.txt b/requirements.txt similarity index 100% rename from requirments.txt rename to requirements.txt diff --git a/run b/run index 6cacf51..b7c22e4 100644 --- a/run +++ b/run @@ -30,6 +30,10 @@ def do_test(): return 0 if r.returncode == 0 else 1 +def do_score(url_file): + from acmecli.cli import main + return main(["score", url_file]) + def main(): if len(sys.argv) < 2: print("Usage: run install|test|score ") diff --git a/src/acmecli.egg-info/SOURCES.txt b/src/acmecli.egg-info/SOURCES.txt index 7ab9ca9..989590b 100644 --- a/src/acmecli.egg-info/SOURCES.txt +++ b/src/acmecli.egg-info/SOURCES.txt @@ -1,5 +1,6 @@ pyproject.toml src/acmecli/__init__.py +src/acmecli/cache.py src/acmecli/cli.py src/acmecli/reporter.py src/acmecli/scoring.py @@ -8,8 +9,16 @@ src/acmecli.egg-info/PKG-INFO src/acmecli.egg-info/SOURCES.txt src/acmecli.egg-info/dependency_links.txt src/acmecli.egg-info/top_level.txt +src/acmecli/handlers/__init__.py +src/acmecli/handlers/github.py src/acmecli/metrics/__init__.py src/acmecli/metrics/base.py +src/acmecli/metrics/heuristic_metrics.py src/acmecli/metrics/license_metric.py tests/test_metrics_contract.py -tests/test_reporter_schema.py \ No newline at end of file +tests/test_reporter_schema.py +requirements.txt +tests/test_cli_flow.py +tests/test_collect_all.py +tests/test_handlers_github.py +tests/test_metrics_behavior.py diff --git a/src/acmecli/__pycache__/__init__.cpython-313.pyc b/src/acmecli/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index bc5983a..0000000 Binary files a/src/acmecli/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/acmecli/__pycache__/types.cpython-313.pyc b/src/acmecli/__pycache__/types.cpython-313.pyc deleted file mode 100644 index 9e236bb..0000000 Binary files a/src/acmecli/__pycache__/types.cpython-313.pyc and /dev/null differ diff --git a/src/acmecli/cache.py b/src/acmecli/cache.py new file mode 100644 index 0000000..eb6e646 --- /dev/null +++ b/src/acmecli/cache.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Optional + + +class MemoryCache: + """In-memory cache that satisfies the Cache protocol used by metrics.""" + + def __init__(self) -> None: + self._store: dict[str, tuple[bytes, Optional[str]]] = {} + + def get(self, key: str) -> bytes | None: + entry = self._store.get(key) + if entry is None: + return None + return entry[0] + + def set(self, key: str, data: bytes, etag: str | None = None) -> None: + self._store[key] = (data, etag) diff --git a/src/acmecli/cli.py b/src/acmecli/cli.py index 6c5d6d3..8ee14fd 100644 --- a/src/acmecli/cli.py +++ b/src/acmecli/cli.py @@ -1,42 +1,151 @@ -import sys +from __future__ import annotations + +import logging +import os +import time from pathlib import Path -from .types import ReportRow +from typing import Dict + +from .handlers import GitHubHandler +from .metrics import heuristic_metrics # noqa: F401 - ensure registration +from .metrics import license_metric # noqa: F401 - ensure registration +from .metrics.base import MetricRunResult, collect_all from .reporter import write_ndjson +from .scoring import ScoringEngine +from .types import ReportRow, TargetSpec + +_LOGGER = logging.getLogger("acmecli") + def _classify(url: str) -> str: u = url.strip().lower() if "huggingface.co/datasets/" in u: return "DATASET" if "github.com/" in u: - return "CODE" + return "MODEL" if "huggingface.co/" in u: return "MODEL" return "CODE" -def _stub_row(name: str) -> ReportRow: - zero = 0.0 - return ReportRow( - name=name, category="MODEL", - net_score=zero, net_score_latency=0, - ramp_up_time=zero, ramp_up_time_latency=0, - bus_factor=zero, bus_factor_latency=0, - performance_claims=zero, performance_claims_latency=0, - license=zero, license_latency=0, - size_score={"raspberry_pi":0.0, "jetson_nano":0.0, "desktop_pc":0.0, "aws_server":0.0}, - size_score_latency=0, - dataset_and_code_score=zero, dataset_and_code_score_latency=0, - dataset_quality=zero, dataset_quality_latency=0, - code_quality=zero, code_quality_latency=0 + +def _setup_logging() -> None: + level = os.getenv("LOG_LEVEL", "0") + path = os.getenv("LOG_FILE") + + try: + numeric_level = int(level) + except ValueError: + numeric_level = 0 + + if numeric_level <= 0 or not path: + logging.basicConfig(level=logging.CRITICAL) + return + + log_level = logging.INFO if numeric_level == 1 else logging.DEBUG + + handler = logging.FileHandler(path, encoding="utf-8") + handler.setLevel(log_level) + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + + _LOGGER.setLevel(log_level) + _LOGGER.handlers.clear() + _LOGGER.addHandler(handler) + + +def _build_spec(url: str) -> TargetSpec: + category = _classify(url) + name = url.rstrip("/").split("/")[-1] or url + source = "GITHUB" if "github.com" in url.lower() else "HUGGINGFACE" + return TargetSpec(url=url, source=source, name=name, category=category) + + +def _clamp(value: float) -> float: + return max(0.0, min(1.0, value)) + + +def _value_for(metrics: Dict[str, float], name: str) -> float: + return _clamp(metrics.get(name, 0.0)) + + +def _latency_for(latencies: Dict[str, int], name: str) -> int: + return int(latencies.get(name, 0)) + + +def _metrics_to_maps(result: MetricRunResult) -> tuple[Dict[str, float], Dict[str, int]]: + value_map: Dict[str, float] = {} + latency_map: Dict[str, int] = {} + for metric in result.values: + value_map[metric.name] = metric.value + latency_map[metric.name] = metric.latency_ms + return value_map, latency_map + + +def _size_average(breakdown: Dict[str, float]) -> float: + if not breakdown: + return 0.0 + return sum(breakdown.values()) / len(breakdown) + + +def _process_url(url: str, handler: GitHubHandler, engine: ScoringEngine) -> None: + spec = _build_spec(url) + if spec.source != "GITHUB": + _LOGGER.info("Skipping unsupported source for URL: %s", url) + return + + result = collect_all(spec, handler) + values, latencies = _metrics_to_maps(result) + + size_breakdown = result.size_breakdown + size_average = _size_average(size_breakdown) + + compute_start = time.perf_counter() + net_score, metric_latency_sum = engine.compute(result.values, size_avg=size_average) + net_latency = metric_latency_sum + int((time.perf_counter() - compute_start) * 1000) + + repo_name = result.meta.get("full_name") or spec.name + + row = ReportRow( + name=repo_name, + category=spec.category, + net_score=_clamp(net_score), + net_score_latency=net_latency, + ramp_up_time=_value_for(values, "ramp_up_time"), + ramp_up_time_latency=_latency_for(latencies, "ramp_up_time"), + bus_factor=_value_for(values, "bus_factor"), + bus_factor_latency=_latency_for(latencies, "bus_factor"), + performance_claims=_value_for(values, "performance_claims"), + performance_claims_latency=_latency_for(latencies, "performance_claims"), + license=_value_for(values, "license"), + license_latency=_latency_for(latencies, "license"), + size_score=size_breakdown, + size_score_latency=_latency_for(latencies, "size"), + dataset_and_code_score=_value_for(values, "dataset_and_code_score"), + dataset_and_code_score_latency=_latency_for(latencies, "dataset_and_code_score"), + dataset_quality=_value_for(values, "dataset_quality"), + dataset_quality_latency=_latency_for(latencies, "dataset_quality"), + code_quality=_value_for(values, "code_quality"), + code_quality_latency=_latency_for(latencies, "code_quality"), ) + write_ndjson(row) + + def main(argv: list[str]) -> int: - # argv pattern: ["score", "/abs/path/URL_FILE"] + if len(argv) < 2: + print("Usage: acmecli score ") + return 1 + + _setup_logging() + handler = GitHubHandler(logger=_LOGGER) + engine = ScoringEngine() + _, url_file = argv lines = Path(url_file).read_text(encoding="utf-8").splitlines() + for raw in lines: url = raw.strip() if not url: continue - if _classify(url) == "MODEL": - write_ndjson(_stub_row(url)) + _process_url(url, handler, engine) + return 0 diff --git a/src/acmecli/handlers/__init__.py b/src/acmecli/handlers/__init__.py new file mode 100644 index 0000000..bc9eb89 --- /dev/null +++ b/src/acmecli/handlers/__init__.py @@ -0,0 +1,3 @@ +from .github import GitHubHandler + +__all__ = ["GitHubHandler"] diff --git a/src/acmecli/handlers/github.py b/src/acmecli/handlers/github.py new file mode 100644 index 0000000..1721124 --- /dev/null +++ b/src/acmecli/handlers/github.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import json +import logging +import os +import re +from typing import Iterable +from urllib import error, request + +from ..types import TargetSpec + +_GITHUB_API = "https://api.github.com" +_RAW_BASE = "https://raw.githubusercontent.com" + + +class GitHubHandler: + """Fetches lightweight repository metadata from GitHub's public API.""" + + def __init__(self, token: str | None = None, *, logger: logging.Logger | None = None) -> None: + self._token = token or os.getenv("GITHUB_TOKEN") or os.getenv("GITHUB_TOKEN_ACME") + self._log = logger or logging.getLogger("acmecli.github") + + # --- SourceHandler protocol ------------------------------------------------- + def resolve_revision(self, url: str) -> str: + owner, repo = _split_repo(url) + data = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}") + if not data: + return "main" + return data.get("default_branch") or "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + owner, repo = _split_repo(spec.url) + repo_url = f"{_GITHUB_API}/repos/{owner}/{repo}" + repo_data = self._fetch_json(repo_url) + if not repo_data: + return { + "owner": owner, + "repo": repo, + "full_name": f"{owner}/{repo}", + "stars": 0, + "forks": 0, + "watchers": 0, + "open_issues": 0, + "default_branch": "main", + "license": "", + "size_kb": 0, + "readme_text": "", + "contributors_count": 0, + "recent_commits": 0, + "topics": [], + "description": "", + "has_wiki": False, + "pushed_at": None, + } + + default_branch = repo_data.get("default_branch") or "main" + readme_text = self._fetch_readme(owner, repo, default_branch) + contributors_count = self._fetch_contributors_count(owner, repo) + recent_commits = self._fetch_recent_commits(owner, repo) + + return { + "owner": owner, + "repo": repo, + "full_name": repo_data.get("full_name") or f"{owner}/{repo}", + "stars": repo_data.get("stargazers_count", 0), + "forks": repo_data.get("forks_count", 0), + "watchers": repo_data.get("subscribers_count", 0), + "open_issues": repo_data.get("open_issues_count", 0), + "default_branch": default_branch, + "license": (repo_data.get("license") or {}).get("name") or "", + "size_kb": repo_data.get("size", 0), + "readme_text": readme_text, + "contributors_count": contributors_count, + "recent_commits": recent_commits, + "topics": repo_data.get("topics", []), + "description": repo_data.get("description") or "", + "has_wiki": bool(repo_data.get("has_wiki", False)), + "pushed_at": repo_data.get("pushed_at"), + "created_at": repo_data.get("created_at"), + "updated_at": repo_data.get("updated_at"), + } + + def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: + meta = self.fetch_meta(spec) + if meta.get("readme_text"): + yield "README.md", meta["readme_text"].encode("utf-8", errors="ignore") + + # --- Internal helpers ------------------------------------------------------- + def _fetch_json(self, url: str) -> dict: + try: + req = request.Request(url, headers=self._headers()) + with request.urlopen(req, timeout=15) as resp: + payload = resp.read() + return json.loads(payload.decode("utf-8")) + except error.HTTPError as exc: + if exc.code == 403: + self._log.debug("GitHub API rate limit hit for %s", url) + else: + self._log.debug("GitHub API error %s for %s", exc.code, url) + except Exception as exc: # pragma: no cover - defensive + self._log.debug("GitHub API fetch failed for %s: %s", url, exc) + return {} + + def _fetch_text(self, url: str) -> str: + try: + req = request.Request(url, headers=self._headers(accept="text/plain")) + with request.urlopen(req, timeout=15) as resp: + payload = resp.read() + return payload.decode("utf-8", errors="ignore") + except Exception: + return "" + + def _fetch_readme(self, owner: str, repo: str, branch: str) -> str: + return self._fetch_text(f"{_RAW_BASE}/{owner}/{repo}/{branch}/README.md") + + def _fetch_contributors_count(self, owner: str, repo: str) -> int: + data = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}/contributors?per_page=100") + if isinstance(data, list): + return len(data) + return 0 + + def _fetch_recent_commits(self, owner: str, repo: str) -> int: + commits = self._fetch_json(f"{_GITHUB_API}/repos/{owner}/{repo}/commits?per_page=30") + if isinstance(commits, list): + return len(commits) + return 0 + + def _headers(self, *, accept: str = "application/vnd.github+json") -> dict[str, str]: + headers = { + "Accept": accept, + "User-Agent": "acmecli/0.0.1", + } + if self._token: + headers["Authorization"] = f"Bearer {self._token}" + return headers + + +def _split_repo(url: str) -> tuple[str, str]: + match = re.search(r"github\.com/([^/]+)/([^/#?]+)", url) + if not match: + raise ValueError(f"Unsupported GitHub URL: {url}") + owner, repo = match.group(1), match.group(2) + if repo.endswith(".git"): + repo = repo[:-4] + return owner, repo diff --git a/src/acmecli/metrics/__init__.py b/src/acmecli/metrics/__init__.py index e69de29..3561957 100644 --- a/src/acmecli/metrics/__init__.py +++ b/src/acmecli/metrics/__init__.py @@ -0,0 +1,21 @@ +from .license_metric import LicenseMetric # noqa: F401 +from .heuristic_metrics import ( # noqa: F401 + BusFactorMetric, + CodeQualityMetric, + DatasetAvailabilityMetric, + DatasetQualityMetric, + PerformanceClaimsMetric, + RampUpMetric, + SizeMetric, +) + +__all__ = [ + "LicenseMetric", + "RampUpMetric", + "BusFactorMetric", + "PerformanceClaimsMetric", + "SizeMetric", + "DatasetAvailabilityMetric", + "DatasetQualityMetric", + "CodeQualityMetric", +] diff --git a/src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc b/src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 63ad76f..0000000 Binary files a/src/acmecli/metrics/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/src/acmecli/metrics/__pycache__/base.cpython-313.pyc b/src/acmecli/metrics/__pycache__/base.cpython-313.pyc deleted file mode 100644 index 2574346..0000000 Binary files a/src/acmecli/metrics/__pycache__/base.cpython-313.pyc and /dev/null differ diff --git a/src/acmecli/metrics/__pycache__/license_metric.cpython-313.pyc b/src/acmecli/metrics/__pycache__/license_metric.cpython-313.pyc deleted file mode 100644 index c89cfa2..0000000 Binary files a/src/acmecli/metrics/__pycache__/license_metric.cpython-313.pyc and /dev/null differ diff --git a/src/acmecli/metrics/base.py b/src/acmecli/metrics/base.py index 1745002..4cabe7a 100644 --- a/src/acmecli/metrics/base.py +++ b/src/acmecli/metrics/base.py @@ -1,7 +1,62 @@ +from __future__ import annotations + +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from typing import List -from ..types import Metric, MetricValue, TargetSpec, SourceHandler, Cache + +from ..cache import MemoryCache +from ..types import Cache, Metric, MetricValue, SourceHandler, TargetSpec REGISTRY: List[Metric] = [] + +@dataclass +class MetricRunResult: + values: list[MetricValue] + size_breakdown: dict[str, float] + total_latency_ms: int + meta: dict + + def register(metric: Metric) -> None: REGISTRY.append(metric) + + +def collect_all( + spec: TargetSpec, + handler: SourceHandler, + cache: Cache | None = None, + max_workers: int | None = None, +) -> MetricRunResult: + """Collect and score all registered metrics in parallel.""" + + if cache is None: + cache = MemoryCache() + + meta = handler.fetch_meta(spec) + cache.set("meta", json.dumps(meta).encode("utf-8")) + + if not REGISTRY: + empty_breakdown = {"raspberry_pi": 0.0, "jetson_nano": 0.0, "desktop_pc": 0.0, "aws_server": 0.0} + return MetricRunResult([], empty_breakdown, 0, meta) + + max_workers = max_workers or min(8, max(1, len(REGISTRY))) + values: list[MetricValue] = [] + size_breakdown: dict[str, float] = {"raspberry_pi": 0.0, "jetson_nano": 0.0, "desktop_pc": 0.0, "aws_server": 0.0} + + def _execute(metric: Metric) -> tuple[MetricValue, dict]: + signals = metric.collect(spec, handler, cache) + value = metric.score(signals) + return value, signals # type: ignore[return-value] + + with ThreadPoolExecutor(max_workers=max_workers) as pool: + future_map = {pool.submit(_execute, metric): metric for metric in REGISTRY} + for future in as_completed(future_map): + value, signals = future.result() + values.append(value) + if value.name == "size": + size_breakdown = signals.get("size_breakdown", size_breakdown) + + total_latency = sum(v.latency_ms for v in values) + return MetricRunResult(values=values, size_breakdown=size_breakdown, total_latency_ms=total_latency, meta=meta) diff --git a/src/acmecli/metrics/heuristic_metrics.py b/src/acmecli/metrics/heuristic_metrics.py new file mode 100644 index 0000000..b0bdc83 --- /dev/null +++ b/src/acmecli/metrics/heuristic_metrics.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import json +import math +import re +import time + +from ..types import Cache, MetricValue, Signals, SourceHandler, TargetSpec +from .base import register + +_KEYWORDS_DATASET = ["dataset", "data set", "training data", "hugging face", "download"] +_KEYWORDS_CODE = ["script", "example", "usage", "cli", "run", "train"] +_KEYWORDS_PERF = ["accuracy", "f1", "precision", "recall", "benchmark", "latency", "throughput", "performance"] + + +def _load_meta(cache: Cache) -> dict: + raw = cache.get("meta") + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + +def _count_keywords(text: str, keywords: list[str]) -> int: + text_lower = text.lower() + return sum(text_lower.count(keyword) for keyword in keywords) + + +def _readme_sections_score(text: str) -> float: + lowered = text.lower() + score = 0.0 + if "installation" in lowered or "getting started" in lowered: + score += 0.3 + if "usage" in lowered or "example" in lowered: + score += 0.3 + if "contributing" in lowered or "license" in lowered: + score += 0.2 + return min(0.8, score) + + +def _device_score(size_mb: float, threshold_mb: float) -> float: + if size_mb <= threshold_mb: + return 1.0 + if size_mb >= threshold_mb * 2: + return 0.0 + return max(0.0, 1.0 - (size_mb - threshold_mb) / threshold_mb) + + +class RampUpMetric: + name = "ramp_up_time" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + "stars": meta.get("stars", 0), + "topics": meta.get("topics", []), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + word_count = len(re.findall(r"\w+", readme)) + base = min(0.6, word_count / 1500) + sections = _readme_sections_score(readme) + topic_bonus = 0.1 if signals.get("topics") else 0.0 + total = min(1.0, base + sections + topic_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class BusFactorMetric: + name = "bus_factor" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "contributors": {"count": meta.get("contributors_count", 0)}, + "recent_commits": meta.get("recent_commits", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + contributors = signals.get("contributors", {}).get("count", 0) + recent_commits = signals.get("recent_commits", 0) + contrib_score = min(1.0, contributors / 5) + freshness = min(0.4, recent_commits / 50) + total = min(1.0, contrib_score + freshness) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class PerformanceClaimsMetric: + name = "performance_claims" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + "stars": meta.get("stars", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + perf_mentions = _count_keywords(readme, _KEYWORDS_PERF) + base = min(0.7, perf_mentions / 5) + stars = signals.get("stars", 0) + stars_bonus = min(0.3, math.log10(stars + 1) / 10) + total = min(1.0, base + stars_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class SizeMetric: + name = "size" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + size_kb = meta.get("size_kb", 0) + size_mb = size_kb / 1024 + breakdown = { + "raspberry_pi": _device_score(size_mb, 50), + "jetson_nano": _device_score(size_mb, 200), + "desktop_pc": _device_score(size_mb, 1024), + "aws_server": _device_score(size_mb, 4096), + } + return { + "size_breakdown": breakdown, + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + breakdown = signals.get("size_breakdown", {}) + if not breakdown: + value = 0.0 + else: + value = sum(breakdown.values()) / len(breakdown) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, max(0.0, min(1.0, value)), latency_ms) + + +class DatasetAvailabilityMetric: + name = "dataset_and_code_score" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + readme = meta.get("readme_text", "") + return { + "readme_text": readme, + "has_examples": any(keyword in readme.lower() for keyword in _KEYWORDS_CODE), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + dataset_mentions = _count_keywords(readme, _KEYWORDS_DATASET) + example_bonus = 0.3 if signals.get("has_examples") else 0.0 + base = min(0.7, dataset_mentions / 5) + total = min(1.0, base + example_bonus) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class DatasetQualityMetric: + name = "dataset_quality" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "readme_text": meta.get("readme_text", ""), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + readme = signals.get("readme_text", "") + mentions = _count_keywords(readme, ["benchmark", "evaluation", "split", "validation", "license"]) + total = min(1.0, mentions / 5) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +class CodeQualityMetric: + name = "code_quality" + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + meta = _load_meta(cache) + return { + "open_issues": meta.get("open_issues", 0), + "recent_commits": meta.get("recent_commits", 0), + "forks": meta.get("forks", 0), + } + + def score(self, signals: Signals) -> MetricValue: + start = time.perf_counter() + open_issues = signals.get("open_issues", 0) + recent_commits = signals.get("recent_commits", 0) + forks = signals.get("forks", 0) + + activity = min(0.6, recent_commits / 50) + balance = 0.4 if open_issues == 0 else max(0.0, 0.4 - (open_issues / 200)) + fork_bonus = min(0.2, forks / 200) + total = max(0.0, min(1.0, activity + balance + fork_bonus)) + latency_ms = int((time.perf_counter() - start) * 1000) + return MetricValue(self.name, total, latency_ms) + + +register(RampUpMetric()) +register(BusFactorMetric()) +register(PerformanceClaimsMetric()) +register(SizeMetric()) +register(DatasetAvailabilityMetric()) +register(DatasetQualityMetric()) +register(CodeQualityMetric()) diff --git a/src/acmecli/metrics/license_metric.py b/src/acmecli/metrics/license_metric.py index 838708a..41ac1c8 100644 --- a/src/acmecli/metrics/license_metric.py +++ b/src/acmecli/metrics/license_metric.py @@ -1,18 +1,51 @@ +from __future__ import annotations + +import json import time -from ..types import Metric, Signals, TargetSpec, SourceHandler, Cache, MetricValue + +from ..types import Cache, Metric, MetricValue, Signals, SourceHandler, TargetSpec from .base import register +_COMPATIBLE_KEYWORDS = ["lgpl", "mit", "bsd", "apache", "mpl", "cc-by"] +_RESTRICTIVE_KEYWORDS = ["gpl", "agpl"] + + +def _load_meta(cache: Cache) -> dict: + raw = cache.get("meta") + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + class LicenseMetric: name = "license" def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: - # Week 1 stub: no network; just return empty signals - return {} + meta = _load_meta(cache) + signals: Signals = { + "license_name": meta.get("license", ""), + "readme_text": meta.get("readme_text", ""), + } + return signals def score(self, signals: Signals) -> MetricValue: t0 = time.perf_counter() - value = 0.0 # TODO: map license presence/compatibility → [0,1] + license_name = (signals.get("license_name") or "").lower() + readme_text = (signals.get("readme_text") or "").lower() + + value = 0.0 + if license_name: + if any(keyword in license_name for keyword in _COMPATIBLE_KEYWORDS): + value = 1.0 + elif any(keyword in license_name for keyword in _RESTRICTIVE_KEYWORDS): + value = 0.3 + else: + value = 0.6 + elif "license" in readme_text: + value = 0.4 + latency_ms = int((time.perf_counter() - t0) * 1000) - return MetricValue(self.name, value, latency_ms) + return MetricValue(self.name, max(0.0, min(1.0, value)), latency_ms) + register(LicenseMetric()) diff --git a/src/acmecli/types.py b/src/acmecli/types.py index 32ec282..8ee207e 100644 --- a/src/acmecli/types.py +++ b/src/acmecli/types.py @@ -1,8 +1,11 @@ -from dataclasses import dataclass, asdict -from typing import Protocol, Iterable, TypedDict, Literal, Optional, Dict +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, Iterable, Literal, NotRequired, Optional, Protocol, TypedDict Category = Literal["MODEL", "DATASET", "CODE"] -Source = Literal["GITHUB", "HUGGINGFACE", "LOCAL"] +Source = Literal["GITHUB", "HUGGINGFACE", "LOCAL"] + @dataclass(frozen=True) class TargetSpec: @@ -12,19 +15,29 @@ class TargetSpec: category: Category revision: Optional[str] = None # commit/tag if known + class Signals(TypedDict, total=False): readme_text: str license_name: str contributors: Dict[str, int] stars: int downloads: int + forks: int + open_issues: int + recent_commits: int + size_breakdown: Dict[str, float] + topics: list[str] + has_examples: bool + documentation_score: float + @dataclass(frozen=True) class MetricValue: - name: str # e.g. "ramp_up_time" - value: float # [0,1] + name: str # e.g. "ramp_up_time" + value: float # [0,1] latency_ms: int + @dataclass(frozen=True) class ReportRow: # Required NDJSON fields + per-metric latencies (values are stubs for now) @@ -40,7 +53,7 @@ class ReportRow: performance_claims_latency: int license: float license_latency: int - size_score: Dict[str, float] # {raspberry_pi, jetson_nano, desktop_pc, aws_server} + size_score: Dict[str, float] # {raspberry_pi, jetson_nano, desktop_pc, aws_server} size_score_latency: int dataset_and_code_score: float dataset_and_code_score_latency: int @@ -49,16 +62,31 @@ class ReportRow: code_quality: float code_quality_latency: int + class SourceHandler(Protocol): - def resolve_revision(self, url: str) -> str: ... - def fetch_meta(self, spec: TargetSpec) -> dict: ... - def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: ... + def resolve_revision(self, url: str) -> str: + ... + + def fetch_meta(self, spec: TargetSpec) -> dict: + ... + + def stream_files(self, spec: TargetSpec, patterns: list[str]) -> Iterable[tuple[str, bytes]]: + ... + class Cache(Protocol): - def get(self, key: str) -> bytes | None: ... - def set(self, key: str, data: bytes, etag: str | None = None) -> None: ... + def get(self, key: str) -> bytes | None: + ... + + def set(self, key: str, data: bytes, etag: str | None = None) -> None: + ... + class Metric(Protocol): name: str - def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: ... - def score(self, signals: Signals) -> MetricValue: ... + + def collect(self, spec: TargetSpec, handler: SourceHandler, cache: Cache) -> Signals: + ... + + def score(self, signals: Signals) -> MetricValue: + ... diff --git a/tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc b/tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc deleted file mode 100644 index ba1c99c..0000000 Binary files a/tests/__pycache__/test_metrics_contract.cpython-313-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc b/tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc deleted file mode 100644 index f2d1c77..0000000 Binary files a/tests/__pycache__/test_reporter_schema.cpython-313-pytest-8.4.2.pyc and /dev/null differ diff --git a/tests/test_cli_flow.py b/tests/test_cli_flow.py new file mode 100644 index 0000000..a1c119d --- /dev/null +++ b/tests/test_cli_flow.py @@ -0,0 +1,97 @@ +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from acmecli.metrics.base import MetricRunResult +from acmecli.types import MetricValue +import acmecli.cli as cli + + +class DummyHandler: + def __init__(self, *args, **kwargs): + pass + + +def test_cli_main_processes_urls(tmp_path, monkeypatch): + url_file = tmp_path / "targets.txt" + url_file.write_text("https://github.com/acme/project\n", encoding="utf-8") + + size_breakdown = { + "raspberry_pi": 0.6, + "jetson_nano": 0.8, + "desktop_pc": 1.0, + "aws_server": 1.0, + } + metric_values = [ + MetricValue("license", 1.0, 10), + MetricValue("ramp_up_time", 0.8, 9), + MetricValue("bus_factor", 0.7, 8), + MetricValue("performance_claims", 0.5, 7), + MetricValue("size", 0.85, 6), + MetricValue("dataset_and_code_score", 0.6, 5), + MetricValue("dataset_quality", 0.4, 4), + MetricValue("code_quality", 0.9, 3), + ] + dummy_result = MetricRunResult( + values=metric_values, + size_breakdown=size_breakdown, + total_latency_ms=sum(m.latency_ms for m in metric_values), + meta={"full_name": "acme/project"}, + ) + + captured_rows = [] + + monkeypatch.setattr(cli, "GitHubHandler", DummyHandler) + monkeypatch.setattr(cli, "collect_all", lambda spec, handler: dummy_result) + monkeypatch.setattr(cli, "write_ndjson", lambda row: captured_rows.append(row)) + + exit_code = cli.main(["score", str(url_file)]) + + assert exit_code == 0 + assert len(captured_rows) == 1 + row = captured_rows[0] + assert row.name == "acme/project" + assert row.net_score == pytest.approx(0.75, abs=1e-6) + assert row.size_score == size_breakdown + assert row.net_score_latency >= sum(m.latency_ms for m in metric_values) + assert row.dataset_quality == 0.4 + assert row.performance_claims == 0.5 + + +def test_classify_variants(): + assert cli._classify('https://huggingface.co/datasets/my-dataset') == 'DATASET' + assert cli._classify('https://github.com/acme/project') == 'MODEL' + assert cli._classify('https://huggingface.co/acme/model') == 'MODEL' + assert cli._classify('https://example.com/something') == 'CODE' + + +def test_setup_logging_creates_file(tmp_path, monkeypatch): + log_file = tmp_path / 'acme.log' + monkeypatch.setenv('LOG_FILE', str(log_file)) + monkeypatch.setenv('LOG_LEVEL', '1') + old_handlers = list(cli._LOGGER.handlers) + cli._LOGGER.handlers.clear() + try: + cli._setup_logging() + assert cli._LOGGER.handlers, 'expected logger to have handlers' + cli._LOGGER.info('hello world') + for handler in list(cli._LOGGER.handlers): + handler.flush() + assert log_file.exists() + finally: + for handler in list(cli._LOGGER.handlers): + handler.close() + cli._LOGGER.removeHandler(handler) + for handler in old_handlers: + cli._LOGGER.addHandler(handler) + monkeypatch.delenv('LOG_FILE', raising=False) + monkeypatch.delenv('LOG_LEVEL', raising=False) + + +def test_build_spec_sets_fields(): + spec = cli._build_spec('https://github.com/acme/tools') + assert spec.source == 'GITHUB' + assert spec.category == 'MODEL' + assert spec.name == 'tools' diff --git a/tests/test_collect_all.py b/tests/test_collect_all.py new file mode 100644 index 0000000..50283a8 --- /dev/null +++ b/tests/test_collect_all.py @@ -0,0 +1,70 @@ +import json + +import pytest + +from acmecli.cache import MemoryCache +from acmecli.metrics.base import MetricRunResult, collect_all +from acmecli.types import MetricValue, TargetSpec + + +class StubHandler: + def __init__(self, meta): + self._meta = meta + + def resolve_revision(self, url: str) -> str: + return "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + return self._meta + + def stream_files(self, spec: TargetSpec, patterns): + return iter(()) + + +def _spec() -> TargetSpec: + return TargetSpec( + url="https://github.com/acme/test", + source="GITHUB", + name="test", + category="MODEL", + ) + + +@pytest.fixture() +def rich_meta(): + return { + "full_name": "acme/test", + "license": "MIT License", + "readme_text": "Installation usage benchmark dataset split validation example", + "stars": 120, + "topics": ["ml"], + "contributors_count": 6, + "recent_commits": 45, + "size_kb": 4096, + "open_issues": 2, + "forks": 40, + } + + +def test_collect_all_returns_metric_values(rich_meta): + handler = StubHandler(rich_meta) + result = collect_all(_spec(), handler, cache=MemoryCache()) + assert isinstance(result, MetricRunResult) + assert result.meta["full_name"] == "acme/test" + assert result.values, "expected registered metrics to produce values" + for metric in result.values: + assert 0.0 <= metric.value <= 1.0 + assert metric.latency_ms >= 0 + assert set(result.size_breakdown.keys()) == { + "raspberry_pi", + "jetson_nano", + "desktop_pc", + "aws_server", + } + + +def test_collect_all_latency_matches_sum(rich_meta): + handler = StubHandler(rich_meta) + result = collect_all(_spec(), handler, cache=MemoryCache()) + summed = sum(m.latency_ms for m in result.values) + assert result.total_latency_ms == summed diff --git a/tests/test_handlers_github.py b/tests/test_handlers_github.py new file mode 100644 index 0000000..949aee6 --- /dev/null +++ b/tests/test_handlers_github.py @@ -0,0 +1,81 @@ +import logging +from types import SimpleNamespace + +import pytest + +from acmecli.handlers.github import GitHubHandler, _split_repo +from acmecli.types import TargetSpec + + +def test_split_repo_removes_suffix(): + owner, repo = _split_repo("https://github.com/acme/project.git") + assert owner == "acme" + assert repo == "project" + + +def test_split_repo_invalid_url(): + with pytest.raises(ValueError): + _split_repo("https://example.com/notgithub") + + +def test_github_handler_fetch_meta(monkeypatch): + repo_data = { + "full_name": "octocat/hello-world", + "stargazers_count": 42, + "forks_count": 7, + "subscribers_count": 3, + "open_issues_count": 1, + "default_branch": "main", + "license": {"name": "MIT License"}, + "size": 2048, + "topics": ["ml"], + "description": "Sample repo", + "has_wiki": True, + "pushed_at": "2024-01-01T00:00:00Z", + "created_at": "2023-01-01T00:00:00Z", + "updated_at": "2024-01-01T00:00:00Z", + } + + def fake_fetch_json(url: str): + if url.endswith("contributors?per_page=100"): + return [{"login": "a"}, {"login": "b"}] + if url.endswith("commits?per_page=30"): + return [{"sha": "1"}, {"sha": "2"}, {"sha": "3"}] + return repo_data + + handler = GitHubHandler(logger=logging.getLogger("test")) + monkeypatch.setattr(handler, "_fetch_json", fake_fetch_json) + monkeypatch.setattr(handler, "_fetch_readme", lambda owner, repo, branch: "# Installation\nUsage") + + spec = TargetSpec( + url="https://github.com/octocat/hello-world", + source="GITHUB", + name="hello-world", + category="MODEL", + ) + + meta = handler.fetch_meta(spec) + assert meta["full_name"] == "octocat/hello-world" + assert meta["stars"] == 42 + assert meta["license"] == "MIT License" + assert meta["contributors_count"] == 2 + assert meta["recent_commits"] == 3 + assert meta["readme_text"].startswith("# Installation") + + +def test_github_handler_default_meta(monkeypatch): + handler = GitHubHandler(logger=logging.getLogger("test")) + monkeypatch.setattr(handler, "_fetch_json", lambda url: {}) + monkeypatch.setattr(handler, "_fetch_readme", lambda owner, repo, branch: "") + + spec = TargetSpec( + url="https://github.com/octocat/unknown", + source="GITHUB", + name="unknown", + category="MODEL", + ) + + meta = handler.fetch_meta(spec) + assert meta["stars"] == 0 + assert meta["license"] == "" + assert meta["full_name"] == "octocat/unknown" diff --git a/tests/test_metrics_behavior.py b/tests/test_metrics_behavior.py new file mode 100644 index 0000000..b892ef6 --- /dev/null +++ b/tests/test_metrics_behavior.py @@ -0,0 +1,154 @@ +import json +import math + +import pytest + +from acmecli.cache import MemoryCache +from acmecli.metrics.heuristic_metrics import ( + BusFactorMetric, + CodeQualityMetric, + DatasetAvailabilityMetric, + DatasetQualityMetric, + PerformanceClaimsMetric, + RampUpMetric, + SizeMetric, +) +from acmecli.metrics.license_metric import LicenseMetric +from acmecli.types import TargetSpec + + +class _DummyHandler: + def __init__(self, meta): + self._meta = meta + + def resolve_revision(self, url: str) -> str: + return "main" + + def fetch_meta(self, spec: TargetSpec) -> dict: + return self._meta + + def stream_files(self, spec: TargetSpec, patterns): + return iter(()) + + +def _cache_with_meta(meta: dict) -> MemoryCache: + cache = MemoryCache() + cache.set("meta", json.dumps(meta).encode("utf-8")) + return cache + + +def _spec() -> TargetSpec: + return TargetSpec( + url="https://github.com/example/repo", + source="GITHUB", + name="repo", + category="MODEL", + ) + + +@pytest.mark.parametrize( + "license_name,readme_text,expected", + [ + ("MIT License", "", 1.0), + ("GNU GPL v3", "", 0.3), + ("", "This project includes a License section", 0.4), + ], +) +def test_license_metric_scores(license_name, readme_text, expected): + metric = LicenseMetric() + signals = {"license_name": license_name, "readme_text": readme_text} + mv = metric.score(signals) + assert mv.name == "license" + assert mv.value == pytest.approx(expected, abs=0.05) + assert mv.latency_ms >= 0 + + +def test_ramp_up_metric_high_score(): + readme = "Installation usage contributing " + "word " * 1600 + meta = {"readme_text": readme, "stars": 50, "topics": ["ml"]} + metric = RampUpMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.9 + assert mv.latency_ms >= 0 + + +def test_ramp_up_metric_low_score(): + meta = {"readme_text": "tiny", "stars": 0, "topics": []} + metric = RampUpMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value < 0.3 + + +def test_bus_factor_metric_range(): + meta = {"contributors_count": 10, "recent_commits": 60} + metric = BusFactorMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert 0.9 <= mv.value <= 1.0 + + +def test_bus_factor_metric_low_activity(): + meta = {"contributors_count": 0, "recent_commits": 0} + metric = BusFactorMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == 0.0 + + +def test_performance_claims_metric_detects_keywords(): + readme = "accuracy accuracy benchmark throughput" + meta = {"readme_text": readme, "stars": 200} + metric = PerformanceClaimsMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.6 + + +def test_performance_claims_metric_without_keywords(): + meta = {"readme_text": "no claims here", "stars": 0} + metric = PerformanceClaimsMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == pytest.approx(0.0) + + +def test_size_metric_average(): + meta = {"size_kb": 1024 * 3} + metric = SizeMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert set(signals["size_breakdown"].keys()) == { + "raspberry_pi", + "jetson_nano", + "desktop_pc", + "aws_server", + } + assert mv.value == pytest.approx(1.0) + + +def test_dataset_availability_metric_examples(): + readme = "Training dataset download script example usage" + meta = {"readme_text": readme} + metric = DatasetAvailabilityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value > 0.6 + + +def test_dataset_quality_metric_mentions(): + readme = "Benchmark evaluation split validation license" + meta = {"readme_text": readme} + metric = DatasetQualityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert mv.value == pytest.approx(1.0) + + +def test_code_quality_metric_balances_signals(): + meta = {"open_issues": 1, "recent_commits": 40, "forks": 80} + metric = CodeQualityMetric() + signals = metric.collect(_spec(), _DummyHandler(meta), _cache_with_meta(meta)) + mv = metric.score(signals) + assert 0.5 < mv.value <= 1.0 diff --git a/tests/test_reporter_schema.py b/tests/test_reporter_schema.py index ba0dbcd..4f45394 100644 --- a/tests/test_reporter_schema.py +++ b/tests/test_reporter_schema.py @@ -14,3 +14,35 @@ def test_reportrow_has_required_fields(): code_quality=0.0, code_quality_latency=0 ) assert row.category == "MODEL" + + +from acmecli.reporter import write_ndjson + + +def test_reporter_prints_ndjson(capsys): + row = ReportRow( + name='demo', + category='MODEL', + net_score=0.5, + net_score_latency=10, + ramp_up_time=0.4, + ramp_up_time_latency=5, + bus_factor=0.3, + bus_factor_latency=4, + performance_claims=0.2, + performance_claims_latency=3, + license=0.9, + license_latency=2, + size_score={'raspberry_pi': 0.5, 'jetson_nano': 0.6, 'desktop_pc': 0.7, 'aws_server': 0.8}, + size_score_latency=1, + dataset_and_code_score=0.6, + dataset_and_code_score_latency=2, + dataset_quality=0.7, + dataset_quality_latency=2, + code_quality=0.8, + code_quality_latency=2, + ) + write_ndjson(row) + captured = capsys.readouterr().out.strip() + assert '"name": "demo"' in captured + assert '"net_score": 0.5' in captured