ffmpeg/libavfilter/dnn_interface.h

/*
 * Copyright (c) 2018 Sergey Lavrushkin
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

/**
 * @file
 * DNN inference engine interface.
 */

#ifndef AVFILTER_DNN_INTERFACE_H
#define AVFILTER_DNN_INTERFACE_H

#include <stdint.h>
#include "libavutil/frame.h"
#include "avfilter.h"

#define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')

typedef enum {
    DNN_TF = 1,
    DNN_OV = 1 << 1,
    DNN_TH = 1 << 2,
    DNN_ONNX = 1 << 3
} DNNBackendType;

typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;

typedef enum {
    DCO_NONE,
    DCO_BGR,
    DCO_RGB,
} DNNColorOrder;

typedef enum {
    DAST_FAIL,              // something wrong
    DAST_EMPTY_QUEUE,       // no more inference result to get
    DAST_NOT_READY,         // all queued inferences are not finished
    DAST_SUCCESS            // got a result frame successfully
} DNNAsyncStatusType;

typedef enum {
    DFT_NONE,
    DFT_PROCESS_FRAME,      // process the whole frame
    DFT_ANALYTICS_DETECT,   // detect from the whole frame
    DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
}DNNFunctionType;

typedef enum {
    DL_NONE,
    DL_NCHW,
    DL_NHWC,
} DNNLayout;

typedef struct DNNData{
    void *data;
    int dims[4];
    // dt and order together decide the color format
    DNNDataType dt;
    DNNColorOrder order;
    DNNLayout layout;
    float scale;
    float mean;
} DNNData;

typedef struct DNNExecBaseParams {
    const char *input_name;
    const char **output_names;
    uint32_t nb_output;
    AVFrame *in_frame;
    AVFrame *out_frame;
} DNNExecBaseParams;

typedef struct DNNExecClassificationParams {
    DNNExecBaseParams base;
    const char *target;
} DNNExecClassificationParams;

typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);

typedef struct DNNModel{
    // Stores FilterContext used for the interaction between AVFrame and DNNData
    AVFilterContext *filter_ctx;
    // Stores function type of the model
    DNNFunctionType func_type;
    // Gets model input information
    // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
    int (*get_input)(struct DNNModel *model, DNNData *input, const char *input_name);
    // Gets model output width/height with given input w/h
    int (*get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height,
                                const char *output_name, int *output_width, int *output_height);
    // set the pre process to transfer data from AVFrame to DNNData
    // the default implementation within DNN is used if it is not provided by the filter
    FramePrePostProc frame_pre_proc;
    // set the post process to transfer data from DNNData to AVFrame
    // the default implementation within DNN is used if it is not provided by the filter
    FramePrePostProc frame_post_proc;
    // set the post process to interpret detect result from DNNData
    DetectPostProc detect_post_proc;
    // set the post process to interpret classify result from DNNData
    ClassifyPostProc classify_post_proc;
} DNNModel;

typedef struct TFOptions{
    const AVClass *clazz;

    char *sess_config;
} TFOptions;

typedef struct OVOptions {
    const AVClass *clazz;

    int input_resizable;
    DNNLayout layout;
    float scale;
    float mean;
} OVOptions;

typedef struct THOptions {
    const AVClass *clazz;
    int optimize;
} THOptions;

#if CONFIG_LIBONNXRUNTIME
typedef struct ONNXOptions {
    const AVClass *clazz;
    int num_threads;
} ONNXOptions;
#endif

typedef struct DNNModule DNNModule;

typedef struct DnnContext {
    const AVClass *clazz;

    DNNModel *model;

    char *model_filename;
    DNNBackendType backend_type;
    char *model_inputname;
    char *model_outputnames_string;
    char *backend_options;
    int async;

    char **model_outputnames;
    uint32_t nb_outputs;
    const DNNModule *dnn_module;

    int nireq;
    int batch_size;
    char *device;
    int device_id;

    AVBufferRef *hw_frames_ctx;
    int zero_copy_negotiated;

#if CONFIG_LIBTENSORFLOW
    TFOptions tf_option;
#endif

#if CONFIG_LIBOPENVINO
    OVOptions ov_option;
#endif
#if CONFIG_LIBTORCH
    THOptions torch_option;
#endif
#if CONFIG_LIBONNXRUNTIME
    ONNXOptions onnx_option;
#endif
} DnnContext;

// Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
struct DNNModule {
    const AVClass clazz;
    DNNBackendType type;
    // Loads model and parameters from given file. Returns NULL if it is not possible.
    DNNModel *(*load_model)(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx);
    // Executes model with specified input and output. Returns the error code otherwise.
    int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
    // Retrieve inference result.
    DNNAsyncStatusType (*get_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
    // Flush all the pending tasks.
    int (*flush)(const DNNModel *model);
    // Frees memory allocated for model.
    void (*free_model)(DNNModel **model);
};

// Initializes DNNModule depending on chosen backend.
const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);

void ff_dnn_init_child_class(DnnContext *ctx);
void *ff_dnn_child_next(DnnContext *obj, void *prev);
const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t backend_mask);

static inline int dnn_get_width_idx_by_layout(DNNLayout layout)
{
    return layout == DL_NHWC ? 2 : 3;
}

static inline int dnn_get_height_idx_by_layout(DNNLayout layout)
{
    return layout == DL_NHWC ? 1 : 2;
}

static inline int dnn_get_channel_idx_by_layout(DNNLayout layout)
{
    return layout == DL_NHWC ? 3 : 1;
}

#endif