CREATE OR REPLACE TABLE `<プロジェクト>.<データセット>.performance_test_1m` AS
SELECT
  -- 適当なID
  GENERATE_UUID() AS id,
  -- 適当な整数と文字列を数カラムずつ用意しておきダミーデータとする
  CAST(FLOOR(RAND() * 10000000) AS INT64) AS i1,
  CAST(FLOOR(RAND() * 10000000) AS INT64) AS i2,
  CAST(FLOOR(RAND() * 10000000) AS INT64) AS i3,
  SHA512(CAST(x * 100000000 AS STRING)) AS s1,
  SHA512(CAST(x * 200000000 AS STRING)) AS s2,
  SHA512(CAST(x * 300000000 AS STRING)) AS s3,
FROM
  -- 100万行
  UNNEST(GENERATE_ARRAY(1, 1000000)) AS x;

from google.cloud import bigquery
PROJECT_ID = "your-project-id"
DATASET_ID = "your_dataset"
TABLE_ID = "performance_test_1m"
def sql_download_to_csv(project_id: str, dataset_id: str, table_id: str, n: int):
    client = bigquery.Client(project=project_id)
    table_ref = f"{project_id}.{dataset_id}.{table_id}"
    query = f"SELECT * FROM `{table_ref}` LIMIT {n}"
    query_job = client.query(query)
    df = query_job.to_dataframe()
    df.to_csv("output.csv", header=False, index=False, encoding="utf-8")
import sys
n = int(sys.argv[1])
sql_download_to_csv(PROJECT_ID, DATASET_ID, TABLE_ID, n)

UserWarning: BigQuery Storage module not found, fetch data with the REST endpoint instead.

from google.cloud import bigquery
def sql_download_to_csv(project_id: str, dataset_id: str, table_id: str, n: int):
    client = bigquery.Client(project=project_id)
    table_ref = f"{project_id}.{dataset_id}.{table_id}"
    query = f"SELECT * FROM `{table_ref}` LIMIT {n}"
    query_job = client.query(query)
    with open("output.csv", "w",  encoding="utf-8") as f:
        for df_chunk in query_job.result().to_dataframe_iterable():
            df_chunk.to_csv(f, header=False, index=False, encoding="utf-8")

from google.cloud import bigquery
from google.cloud import bigquery_storage
def sql_download_to_csv(project_id: str, dataset_id: str, table_id: str, n: int):
    client = bigquery.Client(project=project_id)
    storage_client = bigquery_storage.BigQueryReadClient()  # storage_clientの明示的な生成
    table_ref = f"{project_id}.{dataset_id}.{table_id}"
    query = f"SELECT * FROM `{table_ref}` LIMIT {n}"
    query_job = client.query(query)
    with open("output.csv", "w",  encoding="utf-8") as f:
        # bqstorage_clientに生成したstorage_clientを渡す
        for df_chunk in query_job.result().to_dataframe_iterable(bqstorage_client=storage_client):
            df_chunk.to_csv(f, header=False, index=False, encoding="utf-8")

            create_bqstorage_client (Optional[bool]):
                If ``True`` (default), create a BigQuery Storage API client
                using the default API settings. The BigQuery Storage API
                is a faster way to fetch rows from BigQuery. See the
                ``bqstorage_client`` parameter for more information.
                This argument does nothing if ``bqstorage_client`` is supplied.
                .. versionadded:: 1.24.0

        if not self._should_use_bqstorage(bqstorage_client, create_bqstorage_client):
            create_bqstorage_client = False
            bqstorage_client = None
        record_batch = self.to_arrow(
            progress_bar_type=progress_bar_type,
            bqstorage_client=bqstorage_client,
            create_bqstorage_client=create_bqstorage_client,
        )

        try:
            _versions_helpers.BQ_STORAGE_VERSIONS.try_import(raise_if_error=True)
        except bq_exceptions.BigQueryStorageNotFoundError:
            warnings.warn(
                "BigQuery Storage module not found, fetch data with the REST "
                "endpoint instead."
            )
            return False
        except bq_exceptions.LegacyBigQueryStorageError as exc:
            warnings.warn(str(exc))
            return False

# to_dataframe()版
$ /usr/bin/time -l uv run python simple.py 1000000
       31.73 real        10.81 user         2.48 sys
           901906432  maximum resident set size
                   0  average shared memory size
                   0  average unshared data size
                   0  average unshared stack size
              183905  page reclaims
                3929  page faults
                   0  swaps
                   0  block input operations
                   0  block output operations
                  88  messages sent
               71567  messages received
                   1  signals received
                4406  voluntary context switches
               68786  involuntary context switches
           144949685  instructions retired
            78174140  cycles elapsed
             9044928  peak memory footprint
# to_dataframe_iterable()版
$ /usr/bin/time -l uv run python simple_iter_opt.py 1000000
       25.03 real        10.64 user         1.61 sys
           203636736  maximum resident set size
                   0  average shared memory size
                   0  average unshared data size
                   0  average unshared stack size
               46030  page reclaims
                3933  page faults
                   0  swaps
                   0  block input operations
                   0  block output operations
                 100  messages sent
               54553  messages received
                   1  signals received
                4358  voluntary context switches
              108712  involuntary context switches
           130255972  instructions retired
            67222875  cycles elapsed
             9044928  peak memory footprint