TensorFlow小程序探索实践

 tensorflowjs_converter --input_format=tf_saved_model \
                       --output_format=tfjs_graph_model \
                       --output_node_names='Postprocessor/ExpandDims_1,Postprocessor/Slice' \
                       /Users/echo/projects/vegetables_tf2.3/models/ \              
                       ./web_model

from pathlib import Path
import imghdr
data_dir = "/home/user/datasets/samples/"
image_extensions = [".png", ".jpg"]  # add there all your images file extensions
img_type_accepted_by_tf = ["bmp", "gif", "jpeg", "png"]
for filepath in Path(data_dir).rglob("*"):
    if filepath.suffix.lower() in image_extensions:
        img_type = imghdr.what(filepath)
        if img_type is None:
            print(f"{filepath} is not an image")
        elif img_type not in img_type_accepted_by_tf:
            print(f"{filepath} is a {img_type}, not accepted by TensorFlow")

 --input_format {keras,tfjs_layers_model,tf_saved_model,tf_frozen_model,tf_hub,keras_saved_model}
                        Input format. For "keras", the input path can be one of the two following formats: - A topology+weights
                        combined HDF5 (e.g., generated with `tf.keras.model.save_model()` method). - A weights-only HDF5 (e.g.,
                        generated with Keras Model's `save_weights()` method). For "keras_saved_model", the input_path must point
                        to a subfolder under the saved model folder that is passed as the argument to
                        tf.contrib.save_model.save_keras_model(). The subfolder is generated automatically by tensorflow when
                        saving keras model in the SavedModel format. It is usually named as a Unix epoch time (e.g., 1542212752).
                        For "tf" formats, a SavedModel, frozen model, or TF-Hub module is expected.
  --output_format {tfjs_graph_model,keras_saved_model,keras,tfjs_layers_model}

const imgData = {
      data: new Uint8Array(frame.data),
      height: frame.width,
      width: frame.height,
};

preprocess(imgData) {
    return tf.tidy(() => {
      var tensor = tf.browser.fromPixels(imgData, 1)
      const resized = tf.image.resizeBilinear(tensor, [28, 28]).toFloat()
      const offset = tf.scalar(255.0);
      const normalized = tf.scalar(1.0).sub(resized.div(offset));
      const batched = normalized.expandDims(0)
      return batched
    })
}

tf.tidy(() => {
        const imgData = {
          data: new Uint8Array(frame.data),
          width: frame.width,
          height: frame.height
        }
        const temp = tf.browser.fromPixels(imgData, 4)
        const sliceOptions = getFrameSliceOptions(frame.width, frame.height, this.displaySize.width, this.displaySize.height)

        return temp.slice(sliceOptions.start, sliceOptions.size).resizeBilinear([this.displaySize.height, this.displaySize.width]).asType('int32')
})

tensorflowjs_converter --input_format=keras \
                       --output_format=tfjs_graph_model \
                       --output_node_names='Postprocessor/ExpandDims_1,Postprocessor/Slice' \
                       ./pokemon.h5 \
                       ./web_model

// app.json
"plugins": {
    "tfjsPlugin": {
    "version": "0.2.0",
    "provider": "wx6afed118d9e81df9"
 }

"dependencies": {
    "@tensorflow/tfjs-core": "^1.2.6",
    "@tensorflow/tfjs-layers": "^1.2.2",
    "fetch-wechat": "^0.0.3"
}

// app.js 控制台正确戴银1，2，3，4就是引入成功
var fetchWechat = require('fetch-wechat');
var tf = require('@tensorflow/tfjs-core');
var plugin = requirePlugin('tfjsPlugin');
App({
  onLaunch: function () {
    plugin.configPlugin({
      // polyfill fetch function
      fetchFunc: fetchWechat.fetchFunc(),
      // inject tfjs runtime
      tf,
      // inject webgl backend
      // webgl,
      // provide webgl canvas
      canvas: wx.createOffscreenCanvas()
    });
    console.log('tf',tf);
    tf.tensor([1,2,3,4]).print()
  }
})

const tf = require('@tensorflow/tfjs-core')
const tfl = require('@tensorflow/tfjs-layers')

this.net = await tfl.loadLayersModel('http://192.168.3.5:8080/model2/model.json')
console.log(this.net);
this.net.summary() // 打印模型结构

wx.canvasGetImageData({
              canvasId: 'firstCanvas',
              x: 0,
              y: 0,
              width: 400,
              height: 200,
              success: function (res) {
                const formatRes = that.revertImage(res.data, res.width, res.height)
				// 需要转换一下，否则tf.browser.fromPixels(imgData, 1)会报参数错误
				const imgData = {
					data: new Uint8Array(formatRes.data),
					height: formatRes.width,
					width: formatRes.height,
				};
			}
})

// ...

revertImage(data, width, height) {
    if (!data) return data;
    let dataViews = [];
    let len = width * 4;
    for (let i = 0; i < height; i++) {
      let start = i * width * 4;
      let newBuff = data.slice(start, start + len);
      dataViews.unshift(newBuff);
    }
    let result = this.concatArrayBuffer(...dataViews);
    return {data: result, width, height};
  },
  getMinBox() {
    //get coordinates 
    let coords = drawInfos[0].drawArr
    var coorX = coords.map(function(p) {
        return p.x
    });
    var coorY = coords.map(function(p) {
        return p.y
    });
    //find top left and bottom right corners 
    var min_coords = {
        x: Math.min.apply(null, coorX),
        y: Math.min.apply(null, coorY)
    }
    var max_coords = {
        x: Math.max.apply(null, coorX),
        y: Math.max.apply(null, coorY)
    }
    //return as strucut 
    return {
        min: min_coords,
        max: max_coords
    }
  }

preprocess(imgData) {
    return tf.tidy(() => {
      var tensor = tf.browser.fromPixels(imgData, 1)
      const resized = tf.image.resizeBilinear(tensor, [28, 28]).toFloat()
      const offset = tf.scalar(255.0);
      const normalized = tf.scalar(1.0).sub(resized.div(offset));
      const batched = normalized.expandDims(0)
      return batched
    })
  }

const pred = this.net.predict(formatData).dataSync()

"dependencies": {
    "@tensorflow-models/coco-ssd": "^2.1.0",
    "@tensorflow-models/posenet": "^2.2.1",
    "@tensorflow/tfjs": "^3.18.0",
    "@tensorflow/tfjs-backend-cpu": "^2.7.0",
    "@tensorflow/tfjs-backend-webgl": "^2.7.0",
    "@tensorflow/tfjs-converter": "^2.7.0",
    "@tensorflow/tfjs-core": "^2.7.0",
    "@vant/weapp": "^1.6.1",
    "fetch-wechat": "^0.0.3"
}

var fetchWechat = require('fetch-wechat');
var tf = require('@tensorflow/tfjs-core');
var webgl = require('@tensorflow/tfjs-backend-webgl');
var cpu = require('@tensorflow/tfjs-backend-cpu');
var plugin = requirePlugin('tfjsPlugin');
App({
  onLaunch: function () {
    this.getDeviceInfo();
    tf.ENV.flagRegistry.WEBGL_VERSION.evaluationFn = () => { return 1 };
    plugin.configPlugin({
      // polyfill fetch function
      fetchFunc: fetchWechat.fetchFunc(),
      // inject tfjs runtime
      tf,
      // inject webgl backend
      webgl,
      // inject cpu backend
      cpu,
      // provide webgl canvas
      canvas: wx.createOffscreenCanvas()
    });
    // tf.tensor([1, 2, 3, 4]).print();
  }
})

// index.wxml
<camera class="camera" device-position="back" flash="off" frame-size="medium">
  <canvas class="canvas" canvas-id="ssd"></canvas>
</camera>

// index.js
onReady: function () {
    this.ctx = wx.createCanvasContext('ssd');
    const context = wx.createCameraContext(this);
    this.initClassifier();
    let count = 0;
    const listener = context.onCameraFrame(frame => {
      count++;
      if (count === 2) { // 控制帧数
        if (this.classifier &amp;&amp; this.classifier.isReady()) {
          this.executeClassify(frame);
        }
        count = 0;
      }
    });
    listener.start();
  },

import * as tf from '@tensorflow/tfjs-core'
import * as cocoSsd from '@tensorflow-models/coco-ssd'
// 加载模型
cocoSsd.load({
          modelUrl: SSD_NET_URL,
        })
        .then((model) => {
          console.log('model', model);
          this.ssdNet = model;
          this.ready = true;
        })
        .catch((err) => {
        });

detect(frame) {
    return new Promise((resolve, reject) => {
      const tensor = tf.tidy(() => {
        const imgData = {
          data: new Uint8Array(frame.data),
          width: frame.width,
          height: frame.height
        }
        const temp = tf.browser.fromPixels(imgData, 4)
        const sliceOptions = getFrameSliceOptions(frame.width, frame.height, this.displaySize.width, this.displaySize.height)

        return temp.slice(sliceOptions.start, sliceOptions.size).resizeBilinear([this.displaySize.height, this.displaySize.width]).asType('int32')
      })

      this.ssdNet.detect(tensor).then(res => {
        tensor.dispose()
        resolve(res)
      }).catch(err => {
        console.log(err)
        tensor.dispose()
        reject()
      })
    })
  }

!pip install tensorflow==2.8
!apt install --allow-change-held-packages libcudnn8=8.1.0.77-1+cuda11.2