使用nodejs从google bucket读取多个文件并加载到BQ中

npm install @google-cloud/storage
npm install @google-cloud/bigquery

const { Storage } = require('@google-cloud/storage');

const storage = new Storage({
  projectId: 'your-project-id',
  keyFilename: 'path/to/keyfile.json'
});

const bucket = storage.bucket('your-bucket-name');

const prefix = 'path/to/files/*';

bucket.getFiles({ prefix }, (err, files) => {
  if (err) {
    console.error('Error retrieving files:', err);
    return;
  }

  files.forEach(file => {
    // 处理每个文件
    console.log('File:', file.name);
    
    // 加载文件到BQ中
    loadFileToBQ(file);
  });
});

const { BigQuery } = require('@google-cloud/bigquery');

const bigquery = new BigQuery({
  projectId: 'your-project-id',
  keyFilename: 'path/to/keyfile.json'
});

const datasetId = 'your-dataset-id';
const tableId = 'your-table-id';

const dataset = bigquery.dataset(datasetId);
const table = dataset.table(tableId);

// 创建数据集
dataset.create((err, dataset) => {
  if (err) {
    console.error('Error creating dataset:', err);
    return;
  }

  console.log('Dataset created:', dataset.id);
  
  // 创建表
  const schema = 'your-table-schema';
  
  table.create({ schema }, (err, table) => {
    if (err) {
      console.error('Error creating table:', err);
      return;
    }

    console.log('Table created:', table.id);
  });
});

function loadFileToBQ(file) {
  const metadata = {
    sourceFormat: 'CSV', // 文件格式
    skipLeadingRows: 1, // 跳过文件头行
    schema: 'your-table-schema' // 表结构
  };

  table.load(file, metadata, (err, job) => {
    if (err) {
      console.error('Error loading file:', err);
      return;
    }

    job.on('error', err => {
      console.error('Error loading file:', err);
    });

    job.on('complete', () => {
      console.log('File loaded:', file.name);
    });
  });
}