TensorFlow Serving在Kubernetes中的实践

tensorflow_serving/model_servers/main.cc#L314

int main(int argc, char** argv) {
...
    std::vector<tensorflow::Flag> flag_list = {
        tensorflow::Flag("port", &port, "port to listen on"),
        tensorflow::Flag("enable_batching", &enable_batching, "enable batching"),
        tensorflow::Flag("batching_parameters_file", &batching_parameters_file,
                       "If non-empty, read an ascii BatchingParameters "
                       "protobuf from the supplied file name and use the "
                       "contained values instead of the defaults."),
        tensorflow::Flag("model_config_file", &model_config_file,
                       "If non-empty, read an ascii ModelServerConfig "
                       "protobuf from the supplied file name, and serve the "
                       "models in that file. This config file can be used to "
                       "specify multiple models to serve and other advanced "
                       "parameters including non-default version policy. (If "
                       "used, --model_name, --model_base_path are ignored.)"),
        tensorflow::Flag("model_name", &model_name,
                       "name of model (ignored "
                       "if --model_config_file flag is set"),
        tensorflow::Flag("model_base_path", &model_base_path,
                       "path to export (ignored if --model_config_file flag "
                       "is set, otherwise required)"),
        tensorflow::Flag("file_system_poll_wait_seconds",
                       &file_system_poll_wait_seconds,
                       "interval in seconds between each poll of the file "
                       "system for new model version"),
        tensorflow::Flag("tensorflow_session_parallelism",
                       &tensorflow_session_parallelism,
                       "Number of threads to use for running a "
                       "Tensorflow session. Auto-configured by default."
                       "Note that this option is ignored if "
                       "--platform_config_file is non-empty."),
        tensorflow::Flag("platform_config_file", &platform_config_file,
                       "If non-empty, read an ascii PlatformConfigMap protobuf "
                       "from the supplied file name, and use that platform "
                       "config instead of the Tensorflow platform. (If used, "
                       "--enable_batching is ignored.)")};
...
}

tensorflow_serving/config/model_server_config.proto#L55

// Common configuration for loading a model being served.
message ModelConfig {
  // Name of the model.
  string name = 1;

  // Base path to the model, excluding the version directory.
  // E.g> for a model at /foo/bar/my_model/123, where 123 is the version, the
  // base path is /foo/bar/my_model.
  //
  // (This can be changed once a model is in serving, *if* the underlying data
  // remains the same. Otherwise there are no guarantees about whether the old
  // or new data will be used for model versions currently loaded.)
  string base_path = 2;

  // Type of model.
  // TODO(b/31336131): DEPRECATED. Please use 'model_platform' instead.
  ModelType model_type = 3 [deprecated = true];

  // Type of model (e.g. "tensorflow").
  //
  // (This cannot be changed once a model is in serving.)
  string model_platform = 4;

  reserved 5;

  // Version policy for the model indicating how many versions of the model to
  // be served at the same time.
  // The default option is to serve only the latest version of the model.
  //
  // (This can be changed once a model is in serving.)
  FileSystemStoragePathSourceConfig.ServableVersionPolicy model_version_policy =
      7;

  // Configures logging requests and responses, to the model.
  //
  // (This can be changed once a model is in serving.)
  LoggingConfig logging_config = 6;
}

tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto

message ServableVersionPolicy {
    // Serve the latest versions (i.e. the ones with the highest version
    // numbers), among those found on disk.
    //
    // This is the default policy, with the default number of versions as 1.
    message Latest {
      // Number of latest versions to serve. (The default is 1.)
      uint32 num_versions = 1;
    }

    // Serve all versions found on disk.
    message All {
    }

    // Serve a specific version (or set of versions).
    //
    // This policy is useful for rolling back to a specific version, or for
    // canarying a specific version while still serving a separate stable
    // version.
    message Specific {
      // The version numbers to serve.
      repeated int64 versions = 1;
    }
}

model_config_list: {
	config: {
		name: "mnist",
		base_path: "/tmp/monitored/_model",mnist
		model_platform: "tensorflow",
		model_version_policy: {
		   all: {}
		}
	},
	config: {
		name: "inception",
		base_path: "/tmp/monitored/inception_model",
		model_platform: "tensorflow",
		model_version_policy: {
		   latest: {
		   	num_versions: 2
		   }
		}
	},
	config: {
		name: "mxnet",
		base_path: "/tmp/monitored/mxnet_model",
		model_platform: "tensorflow",
		model_version_policy: {
		   specific: {
		   	versions: 1
		   }
		}
	}
}

Optimized build

It's possible to compile using some platform specific instruction sets (e.g. AVX) that can significantly improve performance. Wherever you see 'bazel build' in the documentation, you can add the flags -c opt --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3 (or some subset of these flags). For example:

bazel build -c opt --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3 tensorflow_serving/...
Note: These instruction sets are not available on all machines, especially with older processors, so it may not work with all flags. You can try some subset of them, or revert to just the basic '-c opt' which is guaranteed to work on all machines.

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: tensorflow-serving
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: "tensorflow-serving"
    spec:
      restartPolicy: Always
      imagePullSecrets:
      - name: harborsecret
      containers:
      - name: tensorflow-serving
        image: registry.vivo.xyz:4443/bigdata_release/tensorflow_serving1.3.0:v0.5
        command: ["/bin/sh", "-c","export CLASSPATH=.:/usr/lib/jvm/java-1.8.0/lib/tools.jar:$(/usr/lib/hadoop-2.6.1/bin/hadoop classpath --glob); /root/tensorflow_model_server --port=8900 --model_name=test_model --model_base_path=hdfs://xx.xx.xx.xx:zz/data/serving_model"]
        ports:
        - containerPort: 8900