【Android -- 开源库】ML Kit 实现数字墨水识别功能

前言

机器学习套件是一个移动 SDK，将 Google 的设备端机器学习专业知识运用于 Android 和 iOS 应用。使用我们强大而易用的 Vision API 和 Natural Language API 解决应用中的常见挑战，或打造全新的用户体验。所有功能均由 Google 一流的机器学习模型提供支持，可免费使用。

学习指南：https://developers.google.cn/ml-kit/vision/digital-ink-recognition/android?hl=zh-cn
Google demo：https://github.com/googlesamples/mlkit/tree/master/android/digitalink

效果图

准备工作

1. 在 app/build.gradle 添加如下依赖

implementation 'com.google.mlkit:digital-ink-recognition:18.0.0'

2. 创建 StrokeManager.java

/**
 * author: Kevin-Dev
 * date: 2023/2/2
 * desc:
 */
public class StrokeManager {
    /** Interface to register to be notified of changes in the recognized content. */
    public interface ContentChangedListener {

        /** This method is called when the recognized content changes. */
        void onContentChanged();
    }

    /** Interface to register to be notified of changes in the status. */
    public interface StatusChangedListener {

        /** This method is called when the recognized content changes. */
        void onStatusChanged();
    }

    /** Interface to register to be notified of changes in the downloaded model state. */
    public interface DownloadedModelsChangedListener {

        /** This method is called when the downloaded models changes. */
        void onDownloadedModelsChanged(Set<String> downloadedLanguageTags);
    }

    @VisibleForTesting
    static final long CONVERSION_TIMEOUT_MS = 1000;
    private static final String TAG = "MLKD.StrokeManager";
    // This is a constant that is used as a message identifier to trigger the timeout.
    private static final int TIMEOUT_TRIGGER = 1;
    // For handling recognition and model downloading.
    private RecognitionTask recognitionTask = null;
    @VisibleForTesting ModelManager modelManager = new ModelManager();
    // Managing the recognition queue.
    private final List<RecognitionTask.RecognizedInk> content = new ArrayList<>();
    // Managing ink currently drawn.
    private Ink.Stroke.Builder strokeBuilder = Ink.Stroke.builder();
    private Ink.Builder inkBuilder = Ink.builder();
    private boolean stateChangedSinceLastRequest = false;
    @Nullable
    private ContentChangedListener contentChangedListener = null;
    @Nullable private StatusChangedListener statusChangedListener = null;
    @Nullable private DownloadedModelsChangedListener downloadedModelsChangedListener = null;

    private boolean triggerRecognitionAfterInput = true;
    private boolean clearCurrentInkAfterRecognition = true;
    private String status = "";

    public void setTriggerRecognitionAfterInput(boolean shouldTrigger) {
        triggerRecognitionAfterInput = shouldTrigger;
    }

    public void setClearCurrentInkAfterRecognition(boolean shouldClear) {
        clearCurrentInkAfterRecognition = shouldClear;
    }

    // Handler to handle the UI Timeout.
    // This handler is only used to trigger the UI timeout. Each time a UI interaction happens,
    // the timer is reset by clearing the queue on this handler and sending a new delayed message (in
    // addNewTouchEvent).
    private final Handler uiHandler =
            new Handler(
                    msg -> {
                        if (msg.what == TIMEOUT_TRIGGER) {
                            Log.i(TAG, "Handling timeout trigger.");
                            commitResult();
                            return true;
                        }
                        // In the current use this statement is never reached because we only ever send
                        // TIMEOUT_TRIGGER messages to this handler.
                        // This line is necessary because otherwise Java's static analysis doesn't allow for
                        // compiling. Returning false indicates that a message wasn't handled.
                        return false;
                    });

    private void setStatus(String newStatus) {
        status = newStatus;
        if (statusChangedListener != null) {
            statusChangedListener.onStatusChanged();
        }
    }

    private void commitResult() {
        if (recognitionTask.done() && recognitionTask.result() != null) {
            content.add(recognitionTask.result());
            setStatus("Successful recognition: " + recognitionTask.result().text);
            if (clearCurrentInkAfterRecognition) {
                resetCurrentInk();
            }
            if (contentChangedListener != null) {
                contentChangedListener.onContentChanged();
            }
            reset();
        }
    }

    public void reset() {
        Log.i(TAG, "reset");
        resetCurrentInk();
        content.clear();
        if (recognitionTask != null && !recognitionTask.done()) {
            recognitionTask.cancel();
        }
        setStatus("");
    }

    private void resetCurrentInk() {
        inkBuilder = Ink.builder();
        strokeBuilder = Ink.Stroke.builder();
        stateChangedSinceLastRequest = false;
    }

    public Ink getCurrentInk() {
        return inkBuilder.build();
    }

    /**
     * This method is called when a new touch event happens on the drawing client and notifies the
     * StrokeManager of new content being added.
     *
     * <p>This method takes care of triggering the UI timeout and scheduling recognitions on the
     * background thread.
     *
     * @return whether the touch event was handled.
     */
    public boolean addNewTouchEvent(MotionEvent event) {
        int action = event.getActionMasked();
        float x = event.getX();
        float y = event.getY();
        long t = System.currentTimeMillis();

        // A new event happened -> clear all pending timeout messages.
        uiHandler.removeMessages(TIMEOUT_TRIGGER);

        switch (action) {
            case MotionEvent.ACTION_DOWN:
            case MotionEvent.ACTION_MOVE:
                strokeBuilder.addPoint(Ink.Point.create(x, y, t));
                break;
            case MotionEvent.ACTION_UP:
                strokeBuilder.addPoint(Ink.Point.create(x, y, t));
                inkBuilder.addStroke(strokeBuilder.build());
                strokeBuilder = Ink.Stroke.builder();
                stateChangedSinceLastRequest = true;
                recognize();
       /* if (triggerRecognitionAfterInput) {
          recognize();
        }*/
                break;
            default:
                // Indicate touch event wasn't handled.
                return false;
        }

        return true;
    }

    // Listeners to update the drawing and status.
    public void setContentChangedListener(ContentChangedListener contentChangedListener) {
        this.contentChangedListener = contentChangedListener;
    }

    public void setStatusChangedListener(StatusChangedListener statusChangedListener) {
        this.statusChangedListener = statusChangedListener;
    }

    public void setDownloadedModelsChangedListener(
            DownloadedModelsChangedListener downloadedModelsChangedListener) {
        this.downloadedModelsChangedListener = downloadedModelsChangedListener;
    }

    public List<RecognitionTask.RecognizedInk> getContent() {
        return content;
    }

    public String getStatus() {
        return status;
    }

    // Model downloading / deleting / setting.

    public void setActiveModel(String languageTag) {
        setStatus(modelManager.setModel(languageTag));
    }

    public Task<Void> deleteActiveModel() {
        return modelManager
                .deleteActiveModel()
                .addOnSuccessListener(unused -> refreshDownloadedModelsStatus())
                .onSuccessTask(
                        status -> {
                            setStatus(status);
                            return Tasks.forResult(null);
                        });
    }

    public Task<Void> download() {
        setStatus("Download started.");
        return modelManager
                .download()
                .addOnSuccessListener(unused -> refreshDownloadedModelsStatus())
                .onSuccessTask(
                        status -> {
                            setStatus(status);
                            return Tasks.forResult(null);
                        });
    }

    // Recognition-related.

    public Task<String> recognize() {

        if (!stateChangedSinceLastRequest || inkBuilder.isEmpty()) {
            setStatus("No recognition, ink unchanged or empty");
            return Tasks.forResult(null);
        }
        if (modelManager.getRecognizer() == null) {
            setStatus("Recognizer not set");
            return Tasks.forResult(null);
        }

        return modelManager
                .checkIsModelDownloaded()
                .onSuccessTask(
                        result -> {
                            if (!result) {
                                setStatus("Model not downloaded yet");
                                return Tasks.forResult(null);
                            }

                            stateChangedSinceLastRequest = false;
                            recognitionTask =
                                    new RecognitionTask(modelManager.getRecognizer(), inkBuilder.build());
                            uiHandler.sendMessageDelayed(
                                    uiHandler.obtainMessage(TIMEOUT_TRIGGER), CONVERSION_TIMEOUT_MS);
                            return recognitionTask.run();
                        });
    }

    public void refreshDownloadedModelsStatus() {
        modelManager
                .getDownloadedModelLanguages()
                .addOnSuccessListener(
                        downloadedLanguageTags -> {
                            if (downloadedModelsChangedListener != null) {
                                downloadedModelsChangedListener.onDownloadedModelsChanged(downloadedLanguageTags);
                            }
                        });
    }
}

3. 创建 ModelManager.java

/**
 * author: Kevin-Dev
 * date: 2023/2/2
 * desc:
 */
public class ModelManager {
    private static final String TAG = "MLKD.ModelManager";
    private DigitalInkRecognitionModel model;
    private DigitalInkRecognizer recognizer;
    final RemoteModelManager remoteModelManager = RemoteModelManager.getInstance();

    public String setModel(String languageTag) {
        // Clear the old model and recognizer.
        model = null;
        if (recognizer != null) {
            recognizer.close();
        }
        recognizer = null;

        // Try to parse the languageTag and get a model from it.
        DigitalInkRecognitionModelIdentifier modelIdentifier;
        try {
            modelIdentifier = DigitalInkRecognitionModelIdentifier.fromLanguageTag(languageTag);
        } catch (MlKitException e) {
            Log.e(TAG, "Failed to parse language '" + languageTag + "'");
            return "";
        }
        if (modelIdentifier == null) {
            return "No model for language: " + languageTag;
        }

        // Initialize the model and recognizer.
        model = DigitalInkRecognitionModel.builder(modelIdentifier).build();
        recognizer =
                DigitalInkRecognition.getClient(DigitalInkRecognizerOptions.builder(model).build());
        Log.i(
                TAG,
                "Model set for language '"
                        + languageTag
                        + "' ('"
                        + modelIdentifier.getLanguageTag()
                        + "').");
        return "Model set for language: " + languageTag;
    }

    public DigitalInkRecognizer getRecognizer() {
        return recognizer;
    }

    public Task<Boolean> checkIsModelDownloaded() {
        return remoteModelManager.isModelDownloaded(model);
    }

    public Task<String> deleteActiveModel() {
        if (model == null) {
            Log.i(TAG, "Model not set");
            return Tasks.forResult("Model not set");
        }
        return checkIsModelDownloaded()
                .onSuccessTask(
                        result -> {
                            if (!result) {
                                return Tasks.forResult("Model not downloaded yet");
                            }
                            return remoteModelManager
                                    .deleteDownloadedModel(model)
                                    .onSuccessTask(
                                            aVoid -> {
                                                Log.i(TAG, "Model successfully deleted");
                                                return Tasks.forResult("Model successfully deleted");
                                            });
                        })
                .addOnFailureListener(e -> Log.e(TAG, "Error while model deletion: " + e));
    }

    public Task<Set<String>> getDownloadedModelLanguages() {
        return remoteModelManager
                .getDownloadedModels(DigitalInkRecognitionModel.class)
                .onSuccessTask(
                        (remoteModels) -> {
                            Set<String> result = new HashSet<>();
                            for (DigitalInkRecognitionModel model : remoteModels) {
                                result.add(model.getModelIdentifier().getLanguageTag());
                            }
                            Log.i(TAG, "Downloaded models for languages:" + result);
                            return Tasks.forResult(result);
                        });
    }

    public Task<String> download() {
        if (model == null) {
            return Tasks.forResult("Model not selected.");
        }
        return remoteModelManager
                .download(model, new DownloadConditions.Builder().build())
                .onSuccessTask(
                        aVoid -> {
                            Log.i(TAG, "Model download succeeded.");
                            return Tasks.forResult("Downloaded model successfully");
                        })
                .addOnFailureListener(e -> Log.e(TAG, "Error while downloading the model: " + e));
    }
}

4. 创建 RecognitionTask.java

/**
 * author: Kevin-Dev
 * date: 2023/2/2
 * desc:
 */
public class RecognitionTask {
    private static final String TAG = "MLKD.RecognitionTask";
    private final DigitalInkRecognizer recognizer;
    private final Ink ink;
    @Nullable
    private RecognizedInk currentResult;
    private final AtomicBoolean cancelled;
    private final AtomicBoolean done;

    public RecognitionTask(DigitalInkRecognizer recognizer, Ink ink) {
        this.recognizer = recognizer;
        this.ink = ink;
        this.currentResult = null;
        cancelled = new AtomicBoolean(false);
        done = new AtomicBoolean(false);
    }

    public void cancel() {
        cancelled.set(true);
    }

    public boolean done() {
        return done.get();
    }

    @Nullable
    public RecognizedInk result() {
        return this.currentResult;
    }

    /** Helper class that stores an ink along with the corresponding recognized text. */
    public static class RecognizedInk {
        public final Ink ink;
        public final String text;

        RecognizedInk(Ink ink, String text) {
            this.ink = ink;
            this.text = text;
        }
    }

    public Task<String> run() {
        Log.i(TAG, "RecoTask.run");
        return recognizer
                .recognize(this.ink)
                .onSuccessTask(
                        result -> {
                            if (cancelled.get() || result.getCandidates().isEmpty()) {
                                return Tasks.forResult(null);
                            }
                            currentResult = new RecognizedInk(ink, result.getCandidates().get(0).getText());
                            Log.i(TAG, "result: " + currentResult.text);
                            done.set(true);
                            return Tasks.forResult(currentResult.text);
                        });
    }
}

自定义 View

1. DrawingView.java

public class DrawingView extends View implements StrokeManager.ContentChangedListener {
  private static final String TAG = "MLKD.DrawingView";
  private static final int STROKE_WIDTH_DP = 3;
  private static final int MIN_BB_WIDTH = 10;
  private static final int MIN_BB_HEIGHT = 10;
  private static final int MAX_BB_WIDTH = 256;
  private static final int MAX_BB_HEIGHT = 256;

  private final Paint recognizedStrokePaint;
  private final TextPaint textPaint;
  private final Paint currentStrokePaint;
  private final Paint canvasPaint;

  private final Path currentStroke;
  private Canvas drawCanvas;
  private Bitmap canvasBitmap;
  private StrokeManager strokeManager;

  public DrawingView(Context context) {
    this(context, null);
  }

  public DrawingView(Context context, AttributeSet attributeSet) {
    super(context, attributeSet);
    currentStrokePaint = new Paint();
    currentStrokePaint.setColor(Color.BLACK);
    currentStrokePaint.setAntiAlias(true);
    // Set stroke width based on display density.
    currentStrokePaint.setStrokeWidth(
            TypedValue.applyDimension(
                    TypedValue.COMPLEX_UNIT_DIP, STROKE_WIDTH_DP, getResources().getDisplayMetrics()));
    currentStrokePaint.setStyle(Paint.Style.STROKE);
    currentStrokePaint.setStrokeJoin(Paint.Join.ROUND);
    currentStrokePaint.setStrokeCap(Paint.Cap.ROUND);

    recognizedStrokePaint = new Paint(currentStrokePaint);
    recognizedStrokePaint.setColor(Color.BLACK);

    textPaint = new TextPaint();
    textPaint.setColor(Color.GREEN);

    currentStroke = new Path();
    canvasPaint = new Paint(Paint.DITHER_FLAG);
  }

  private static Rect computeBoundingBox(Ink ink) {
    float top = Float.MAX_VALUE;
    float left = Float.MAX_VALUE;
    float bottom = Float.MIN_VALUE;
    float right = Float.MIN_VALUE;
    for (Ink.Stroke s : ink.getStrokes()) {
      for (Ink.Point p : s.getPoints()) {
        top = Math.min(top, p.getY());
        left = Math.min(left, p.getX());
        bottom = Math.max(bottom, p.getY());
        right = Math.max(right, p.getX());
      }
    }
    float centerX = (left + right) / 2;
    float centerY = (top + bottom) / 2;
    Rect bb = new Rect((int) left, (int) top, (int) right, (int) bottom);
    // Enforce a minimum size of the bounding box such that recognitions for small inks are readable
    bb.union(
            (int) (centerX - MIN_BB_WIDTH / 2),
            (int) (centerY - MIN_BB_HEIGHT / 2),
            (int) (centerX + MIN_BB_WIDTH / 2),
            (int) (centerY + MIN_BB_HEIGHT / 2));
    // Enforce a maximum size of the bounding box, to ensure Emoji characters get displayed
    // correctly
    /*if (bb.width() > MAX_BB_WIDTH) {
      bb.set(bb.centerX() - MAX_BB_WIDTH / 2, bb.top, bb.centerX() + MAX_BB_WIDTH / 2, bb.bottom);
    }
    if (bb.height() > MAX_BB_HEIGHT) {
      bb.set(bb.left, bb.centerY() - MAX_BB_HEIGHT / 2, bb.right, bb.centerY() + MAX_BB_HEIGHT / 2);
    }*/
    return bb;
  }

  void setStrokeManager(StrokeManager strokeManager) {
    this.strokeManager = strokeManager;
  }

  @Override
  protected void onSizeChanged(int width, int height, int oldWidth, int oldHeight) {
    Log.i(TAG, "onSizeChanged");
    canvasBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
    drawCanvas = new Canvas(canvasBitmap);
    invalidate();
  }

  public void redrawContent() {
    clear();
    Ink currentInk = strokeManager.getCurrentInk();
    drawInk(currentInk, currentStrokePaint);

    List<RecognitionTask.RecognizedInk> content = strokeManager.getContent();
    for (RecognitionTask.RecognizedInk ri : content) {
      drawInk(ri.ink, recognizedStrokePaint);
      final Rect bb = computeBoundingBox(ri.ink);
      drawTextIntoBoundingBox(ri.text, bb, textPaint);
    }
    invalidate();
  }

  private void drawTextIntoBoundingBox(String text, Rect bb, TextPaint textPaint) {
    final float arbitraryFixedSize = 20.f;
    // Set an arbitrary text size to learn how high the text will be.
    textPaint.setTextSize(arbitraryFixedSize);
    textPaint.setTextScaleX(1.f);

    // Now determine the size of the rendered text with these settings.
    Rect r = new Rect();
    textPaint.getTextBounds(text, 0, text.length(), r);

    // Adjust height such that target height is met.
    float textSize = arbitraryFixedSize * (float) bb.height() / (float) r.height();
    textPaint.setTextSize(textSize);

    // Redetermine the size of the rendered text with the new settings.
    textPaint.getTextBounds(text, 0, text.length(), r);

    // Adjust scaleX to squeeze the text.
    textPaint.setTextScaleX((float) bb.width() / (float) r.width());

    // And finally draw the text.
    drawCanvas.drawText(text, bb.left, bb.bottom, textPaint);
  }

  private void drawInk(Ink ink, Paint paint) {
    for (Ink.Stroke s : ink.getStrokes()) {
      drawStroke(s, paint);
    }
  }

  private void drawStroke(Ink.Stroke s, Paint paint) {
    Log.i(TAG, "drawstroke");
    Path path = null;
    for (Ink.Point p : s.getPoints()) {
      if (path == null) {
        path = new Path();
        path.moveTo(p.getX(), p.getY());
      } else {
        path.lineTo(p.getX(), p.getY());
      }
    }
    drawCanvas.drawPath(path, paint);
  }

  public void clear() {
    currentStroke.reset();
    onSizeChanged(
            canvasBitmap.getWidth(),
            canvasBitmap.getHeight(),
            canvasBitmap.getWidth(),
            canvasBitmap.getHeight());
  }

  @Override
  protected void onDraw(Canvas canvas) {
    canvas.drawBitmap(canvasBitmap, 0, 0, canvasPaint);
    canvas.drawPath(currentStroke, currentStrokePaint);
  }

  @Override
  public boolean onTouchEvent(MotionEvent event) {
    int action = event.getActionMasked();
    float x = event.getX();
    float y = event.getY();

    switch (action) {
      case MotionEvent.ACTION_DOWN:
        currentStroke.moveTo(x, y);
        break;
      case MotionEvent.ACTION_MOVE:
        currentStroke.lineTo(x, y);
        break;
      case MotionEvent.ACTION_UP:
        currentStroke.lineTo(x, y);
        drawCanvas.drawPath(currentStroke, currentStrokePaint);
        currentStroke.reset();
        break;
      default:
        break;
    }
    strokeManager.addNewTouchEvent(event);
    invalidate();
    return true;
  }

  @Override
  public void onContentChanged() {
    redrawContent();
  }
}

2. StatusTextView.java

public class StatusTextView extends TextView implements StrokeManager.StatusChangedListener {

  private StrokeManager strokeManager;

  public StatusTextView(@NonNull Context context) {
    super(context);
  }

  public StatusTextView(Context context, AttributeSet attributeSet) {
    super(context, attributeSet);
  }

  @Override
  public void onStatusChanged() {
    this.setText(this.strokeManager.getStatus());
  }

  void setStrokeManager(StrokeManager strokeManager) {
    this.strokeManager = strokeManager;
  }
}

使用

1. 布局文件

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:orientation="vertical" android:layout_width="match_parent"
    android:layout_height="match_parent">

    <com.kjd.gesturedemo.ai.DrawingView
        android:id="@+id/drawing_view"
        android:layout_width="match_parent"
        android:layout_height="200dp"
        android:layout_weight="1"
        android:background="#80FFFFFF" />
    <FrameLayout
        android:layout_width="match_parent"
        android:layout_height="wrap_content">

        <LinearLayout
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:orientation="vertical">
            <com.kjd.gesturedemo.ai.StatusTextView
                android:id="@+id/status_text_view"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:text="Status text..."
                android:textIsSelectable="true" />
            <LinearLayout
                android:layout_width="match_parent"
                android:layout_height="60dp"
                android:orientation="horizontal">

                <Button
                    android:id="@+id/clear_button"
                    android:layout_width="0dp"
                    android:layout_height="wrap_content"
                    android:layout_weight="1"
                    android:onClick="clearClick"
                    android:text="清除" />
            </LinearLayout>
        </LinearLayout>
    </FrameLayout>

</LinearLayout>

2. RecognitionActivity.java文章来源地址https://www.toymoban.com/news/detail-490347.html

public class RecognitionActivity extends AppCompatActivity implements StrokeManager.DownloadedModelsChangedListener {
    private static final String TAG = "MLKDI.Activity";
    
    @VisibleForTesting
    final StrokeManager strokeManager = new StrokeManager();

    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_recognition);
        
        DrawingView drawingView = findViewById(R.id.drawing_view);
        StatusTextView statusTextView = findViewById(R.id.status_text_view);
        drawingView.setStrokeManager(strokeManager);
        statusTextView.setStrokeManager(strokeManager);

        strokeManager.setStatusChangedListener(statusTextView);
        strokeManager.setContentChangedListener(drawingView);
        strokeManager.setActiveModel("zh-Hani-CN");
        strokeManager.setDownloadedModelsChangedListener(this);
        strokeManager.setClearCurrentInkAfterRecognition(true);
        strokeManager.setTriggerRecognitionAfterInput(false);

        strokeManager.download();

        strokeManager.recognize();
        
        strokeManager.refreshDownloadedModelsStatus();
        
        strokeManager.reset();
    }

    public void clearClick(View v) {
        strokeManager.reset();
        DrawingView drawingView = findViewById(R.id.drawing_view);
        drawingView.clear();
    }

    @Override
    public void onDownloadedModelsChanged(Set<String> downloadedLanguageTags) {

    }

}

到了这里，关于【Android -- 开源库】ML Kit 实现数字墨水识别功能的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！