check_accuracy() — pytorch Function Reference

Architecture documentation for the check_accuracy() function in check_accuracy.py from the pytorch codebase.

Function python DynamoBenchmarks Microbenchmarking calls 1 called by 1

Entity Profile

DynamoBenchmarks→ Microbenchmarking→ check_accuracy() — pytorch Function Reference

Dependency Diagram

graph TD
  858864ce_33a8_9654_b5b1_91feeb2494e7["check_accuracy()"]
  fbb07b79_5d92_602c_7694_3dbe255e2317["main()"]
  fbb07b79_5d92_602c_7694_3dbe255e2317 -->|calls| 858864ce_33a8_9654_b5b1_91feeb2494e7
  38738e1d_d593_820a_ba60_bd219308d7ba["get_field()"]
  858864ce_33a8_9654_b5b1_91feeb2494e7 -->|calls| 38738e1d_d593_820a_ba60_bd219308d7ba
  style 858864ce_33a8_9654_b5b1_91feeb2494e7 fill:#6366f1,stroke:#818cf8,color:#fff

Relationship Graph

Source Code

benchmarks/dynamo/check_accuracy.py lines 30–113

def check_accuracy(actual_csv, expected_csv, expected_filename):
    failed = []
    improved = []

    if "rocm" in expected_filename:
        flaky_models.update(
            {
                "Background_Matting",
                "mnasnet1_0",
                "llava",
                "repvgg_a2",
                "resnet152",
                "resnet18",
                "resnet50",
                "stable_diffusion_unet",
                "torchrec_dlrm",
                "shufflenet_v2_x1_0",
                "vgg16",
                "BERT_pytorch",
                # LLM
                "google/gemma-2-2b",
                "tts_angular",  # RuntimeError: Cannot access data pointer of Tensor
            }
        )

    for model in actual_csv["name"]:
        accuracy = get_field(actual_csv, model, "accuracy")
        expected_accuracy = get_field(expected_csv, model, "accuracy")

        if accuracy is None:
            status = "MISSING_ACCURACY:"
            failed.append(model)
        elif expected_accuracy is None:
            status = "MISSING_EXPECTED:"
            failed.append(model)
        elif accuracy == expected_accuracy:
            status = "PASS" if expected_accuracy == "pass" else "XFAIL"
            print(f"{model:34}  {status}")
            continue
        elif model in flaky_models:
            if accuracy == "pass":
                # model passed but marked xfailed
                status = "PASS_BUT_FLAKY:"
            else:
                # model failed but marked passe
                status = "FAIL_BUT_FLAKY:"
        elif accuracy != "pass":
            status = "FAIL:"
            failed.append(model)
        else:
            status = "IMPROVED:"
            improved.append(model)
        print(
            f"{model:34}  {status:9} accuracy={accuracy}, expected={expected_accuracy}"
        )

    msg = ""
    if failed or improved:
        if failed:
            msg += textwrap.dedent(
                f"""
            Error: {len(failed)} models have accuracy status regressed:
                {" ".join(failed)}

            """
            )
        if improved:
            msg += textwrap.dedent(
                f"""
            Improvement: {len(improved)} models have accuracy status improved:
                {" ".join(improved)}

            """
            )
        sha = os.getenv("SHA1", "{your CI commit sha}")
        msg += textwrap.dedent(
            f"""
        If this change is expected, you can update `{expected_filename}` to reflect the new baseline.
        from pytorch/pytorch root, run
        `python benchmarks/dynamo/ci_expected_accuracy/update_expected.py {sha}`
        and then `git add` the resulting local changes to expected CSVs to your commit.
        """
        )
    return failed or improved, msg