download_artifacts_and_extract_csvs() — pytorch Function Reference

Source Code

benchmarks/dynamo/ci_expected_accuracy/update_expected.py lines 206–231

def download_artifacts_and_extract_csvs(urls):
    dataframes = {}
    with ThreadPoolExecutor(max_workers=16) as executor:
        futures = {
            executor.submit(download_single_artifact, suite, shard, url_candidates): (
                suite,
                shard,
                url_candidates,
            )
            for (suite, shard), url_candidates in urls.items()
        }
        for future in as_completed(futures):
            suite, shard, url_candidates = futures[future]
            suite_result, shard_result, result = future.result()
            if result is None:
                print(
                    f"Unable to download any artifact for {suite} shard {shard}, tried {len(url_candidates)} URLs"
                )
            else:
                for (s, phase), df in result.items():
                    prev_df = dataframes.get((s, phase), None)
                    dataframes[(s, phase)] = (
                        pd.concat([prev_df, df]) if prev_df is not None else df
                    )

    return dataframes