如何在不使用Bulkloader的情况下将数据上传到GAE

pip install google-cloud-datastore-bulkloader

bulkloader load --dataset_id=YOUR_DATASET_ID --input_file=YOUR_DATA_FILE

from google.cloud import datastore_v1
​
# 创建 Bulkloader 客户端。
bulkloader = datastore_v1.DatastoreAdminClient()
​
# 创建数据集。
dataset = datastore_v1.types.Dataset(
    name=datastore_v1.DatastoreAdminClient.dataset_path(
        "YOUR_PROJECT_ID", "YOUR_DATASET_ID"
    )
)
bulkloader.create_dataset(request={"dataset": dataset})
​
# 准备数据文件。
data_file = "path/to/your/data/file.csv"
​
# 加载数据。
operation = bulkloader.load_data(
    request={
        "parent": datastore_v1.DatastoreAdminClient.dataset_path(
            "YOUR_PROJECT_ID", "YOUR_DATASET_ID"
        ),
        "input_url": f"gs://{bucket_name}/{file_name}",
        "name": "YOUR_OPERATION_NAME",
    }
)
​
# 等待加载操作完成。
operation.result(300)

import com.google.cloud.datastore.bulkloader.v1.BulkLoaderServiceClient;
import com.google.cloud.datastore.bulkloader.v1.DataImport;
import com.google.cloud.datastore.bulkloader.v1.DataImport.DataFile;
import com.google.cloud.datastore.bulkloader.v1.DataImport.EntityFilter;
import com.google.cloud.datastore.bulkloader.v1.ImportEntitiesRequest;
import com.google.cloud.datastore.bulkloader.v1.OperationMetadata;
import com.google.datastore.v1.DatastoreName;
import com.google.datastore.v1.Key;
import com.google.protobuf.ByteString;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
​
public class ImportData {
​
  public static void main(String[] args)
      throws IOException, NoSuchAlgorithmException, ExecutionException, InterruptedException, TimeoutException {
    // TODO(developer): Replace these variables before running the sample.
    String projectId = "your-project-id";
    String datasetId = "your-dataset-id";
    String gcsSourceUri = "gs://source-bucket/source.csv";
​
    importData(projectId, datasetId, gcsSourceUri);
  }
​
  // Imports data to the specified dataset.
  public static void importData(String projectId, String datasetId, String gcsSourceUri)
      throws IOException, NoSuchAlgorithmException, ExecutionException, InterruptedException, TimeoutException {
    try (BulkLoaderServiceClient client = BulkLoaderServiceClient.create()) {
      String datasetName = DatastoreName.of(projectId, datasetId).toString();
​
      // Data source information.
      DataFile dataFile = DataFile.newBuilder().setGcsSource(gcsSourceUri).build();
​
      // Import with an empty filter imports all entities in the file.
      EntityFilter entityFilter = EntityFilter.newBuilder().build();
​
      // Compute the hash of the source data for the DataImport.
      String hash = computeHash(Paths.get(gcsSourceUri).toFile());
      DataImport dataImport =
          DataImport.newBuilder()
              .setGcsSource(gcsSourceUri)
              .setEntityFilter(entityFilter)
              .setSourceDataHash(ByteString.copyFromUtf8(hash))
              .build();
​
      ImportEntitiesRequest request =
          ImportEntitiesRequest.newBuilder()
              .setParent(datasetName)
              .setDataImport(dataImport)
              .build();
​
      // Start an import job.
      OperationMetadata operation = client.importEntitiesAsync(request).get(10, TimeUnit.MINUTES);
​
      // Block until the operation completes.
      System.out.println("Waiting for operation to complete: " + operation.getName());
      operation = client.getOperation(operation.getName());
​
      // Check the status of the import job.
      if (!operation.getDone()) {
        System.out.println("Import job failed: " + operation.getErrorMessage());
        return;
      }
      System.out.println("Import job completed successfully.");
    }
  }
​
  // Computes the MD5 hash of the data in the file.
  private static String computeHash(java.io.File file) throws IOException, NoSuchAlgorithmException {
    MessageDigest md = MessageDigest.getInstance("MD5");
    try (FileInputStream fis = new FileInputStream(file)) {
      byte[] buffer = new byte[8192];
      int length;
      while ((length = fis.read(buffer)) != -1) {
        md.update(buffer, 0, length);
      }
    }
    return toHex(md.digest());
  }
​
  // Converts a byte array to a hex string.
  private static String toHex(byte[] bytes) {
    StringBuilder sb = new StringBuilder();
    for (byte b : bytes) {
      sb.append(String.format("%02x", b & 0xFF));
    }
    return sb.toString();
  }
}