Halcon深度学习-目标检测-Rectangle1

Halcon深度学习-目标检测-Rectangle1

备注：Halcon版本19.11+；

整体代码预览：

dev_update_off ()

*读数据
read_dict ('Gen_Xml_To_dict.hdict', [], [], DictHandle)

*Xml_To_Dict (Xml格式转dict)
Xml_To_Dict_DeepBool ('Train_Images/JPEGImages', 'Train_Images/Temp', 'Xml', DictHandle)

*Dict_To_Xml（dict格式转xml)
* Dict_To_Xml_DeepBool (DictHandle, 'Voc', 'Temp')
* 
*by_num (通过数量定义滑动窗口）
* Cut_Images_bynum (DictHandle, 2, 2, 'png', 255, DictHandle)
* 
*by_size (通过尺寸定义滑动窗口）
* Cut_Images_bysize (DictHandle, 800, 800, 'png', 255, DictHandle)

*数据预处理
Data_preprocessing_Rectangle1 (224, 224, 1, 'ResNet101', DictHandle, 0, 'bmp', 255)

*训练
Data_Train_Rectangle1 (1, 0.00001, 0.9, 200, 0.0001, 1, false, 'gpu', [], [])

*模型评估
Data_Evaluate_Rectangle1 (0.5, 2, true, 0.7, 0.2, 0.5)

*推理初始化
Data_Infer_Init_Rectangle1 (1, 0.5, true, 0.2, 0.7, DLModelHandle, DLPreprocessParam)

file_exists ('Data_Infer_Result', FileExists)
if (not FileExists)
    make_dir ('Data_Infer_Result')
endif

*Data_Infer_Online
dev_get_window (WindowHandle_now)
dev_clear_window ()

get_dict_tuple (DictHandle,'class_names' , class_names)

list_files ('pcb', ['files','follow_links'], ImageFiles)
tuple_regexp_select (ImageFiles, ['\\.(tif|tiff|gif|bmp|jpg|jpeg|jp2|png|pcx|pgm|ppm|pbm|xwd|ima|hobj)$','ignore_case'], ImageFiles)
for Index := 0 to |ImageFiles| - 1 by 1
    read_image (Image, ImageFiles[Index])
    Data_Infer_OnLine_Rectangle1 (Image, Rectangle, DictHandle, DLPreprocessParam, DLModelHandle, bbox_row1, bbox_col1, bbox_row2, bbox_col2, bbox_class_id, bbox_confidence)
    ifelse (|bbox_class_id|)
        gen_rectangle1 (Rectangle, bbox_row1, bbox_col1, bbox_row2, bbox_col2)
        dev_clear_window ()
        dev_display (Image)
        boundary (Rectangle, Rectangle, 'inner')
        dev_disp_text (class_names[bbox_class_id]+'  :  '+bbox_confidence, 'image', bbox_row1, bbox_col1, 'red', 'box', 'false')
    else
        throw (['未检测到目标'])
    endif
    dump_window_image (Image1, WindowHandle_now)
    tuple_split (ImageFiles[Index], '\\', Substrings)
    write_image (Image1, 'jpg', 0, 'Data_Infer_Result/'+Substrings[|Substrings|-1])
endfor

1、数据预处理：(Data_preprocessing_Rectangle1):

dev_update_off ()

* dev_get_window (WindowHandle_now)
* if (WindowHandle_now == -1)
*     dev_open_window (0, 0, 700, 700, 'white', WindowHandle)
* endif
*********************************************************************
*数据增强
if (data_enhanced > 0)
    *判断是否灰度化图片
    ifelse (ImageNumChannels == 1)
        Is_GrayImage := true
    else
         Is_GrayImage := false
    endif
    *加入数据增强
    *目前目标检测数据增强含有bug，无标签图片禁止参与数据增强
    data_enhanced_detection_rectangle1 (DLDataset, images_fotmat, imags_fill, data_enhanced, Is_GrayImage, DLDataset)
    write_dict (DLDataset, 'Gen_Xml_To_dict_New', [], [])
endif
*********************************************************************
*预处理前删除上次数据
file_exists ('detect_data', FileExists1)
if (FileExists1)
    remove_dir_recursively ('detect_data')
endif

*模型存放文件
file_exists ('New_Model', FileExists)
if (not FileExists)
    make_dir ('New_Model')
endif
*********************************************************************
*模型类型
Backbone := ''
MinLevel := 2
if(Model_type == 'alexnet')
     Backbone := 'pretrained_dl_classifier_alexnet.hdl'
     MaxLevel := 4
endif
if (Model_type == 'compact')
    Backbone := 'pretrained_dl_classifier_compact.hdl'
    MaxLevel := 4
endif
if (Model_type == 'enhanced')
    Backbone := 'pretrained_dl_classifier_enhanced.hdl'
    MaxLevel := 5
endif
if (Model_type == 'resnet50')
    Backbone := 'pretrained_dl_classifier_resnet50.hdl'
    MaxLevel := 5
endif

*兼容不同的版本模型
if (Model_type == 'mobilenet_v2')
    get_system ('version', version)
    ifelse (number(version) >= 21.05)
        Backbone := 'pretrained_dl_classifier_mobilenet_v2.hdl'
        MaxLevel := 4
    else
        throw (['21.05以下版本不支持：pretrained_dl_classifier_mobilenet_v2模型'])
    endif
endif
*加载自定义模型
try
    if (Backbone == '')
        Backbone := Model_type+'.hdl'
        MaxLevel := 5
    endif
catch (Exception)
    throw (['Model_type error :'] + Exception)
endtry
************************************************************************
Capacity := 'medium'
TrainingPercent := 75
ValidationPercent := 15

ExampleDataDir := 'detect_data'

DLModelFileName := ExampleDataDir + '/pretrained_dl_model_detection.hdl'

DataDirectory := ExampleDataDir + '/dldataset' 

PreprocessParamFileName := 'New_Model/dl_preprocess_param.hdict'

*随机种子
set_system ('seed_rand', 42)

file_exists (ExampleDataDir, FileExists)
if (not FileExists)
    make_dir (ExampleDataDir)
endif

*分割数据集
split_dl_dataset_Deep_Bool (DLDataset, TrainingPercent, ValidationPercent, [])

create_dict (GenParam)
set_dict_tuple (GenParam, 'split', 'train')

determine_dl_model_detection_param (DLDataset, ImageWidth, ImageHeight, GenParam, DLDetectionModelParam)

* get_dict_tuple (DLDetectionModelParam, 'min_level', MinLevel)
* get_dict_tuple (DLDetectionModelParam, 'max_level', MaxLevel)
get_dict_tuple (DLDetectionModelParam, 'anchor_num_subscales', AnchorNumSubscales)
get_dict_tuple (DLDetectionModelParam, 'anchor_aspect_ratios', AnchorAspectRatios)

create_dict (DLModelDetectionParam)
set_dict_tuple (DLModelDetectionParam, 'image_width', ImageWidth)
set_dict_tuple (DLModelDetectionParam, 'image_height', ImageHeight)
set_dict_tuple (DLModelDetectionParam, 'image_num_channels', ImageNumChannels)
set_dict_tuple (DLModelDetectionParam, 'min_level', MinLevel)
set_dict_tuple (DLModelDetectionParam, 'max_level', MaxLevel)
set_dict_tuple (DLModelDetectionParam, 'anchor_num_subscales', AnchorNumSubscales)
set_dict_tuple (DLModelDetectionParam, 'anchor_aspect_ratios', AnchorAspectRatios)
set_dict_tuple (DLModelDetectionParam, 'capacity', Capacity)

get_dict_tuple (DLDataset, 'class_ids', ClassIDs)
set_dict_tuple (DLModelDetectionParam, 'class_ids', ClassIDs)

*DictHandle
tuple_length (ClassIDs, NumClasses)

create_dl_model_detection (Backbone, NumClasses, DLModelDetectionParam, DLModelHandle)

write_dl_model (DLModelHandle, DLModelFileName)
 
create_dl_preprocess_param_from_model (DLModelHandle, 'none', 'full_domain', [], [], [], DLPreprocessParam)
 
create_dict (GenParam)
set_dict_tuple (GenParam, 'overwrite_files', true)

*halcon预处理
preprocess_dl_dataset (DLDataset, DataDirectory, DLPreprocessParam, GenParam, DLDatasetFilename)

write_dict (DLPreprocessParam, PreprocessParamFileName, [], [])

return ()

2、训练：（Data_Train_Rectangle1 ）:

dev_update_off ()

ExampleDataDir := 'detect_data'
*选择是否累加训练方案
ifelse (OldModel)
    InitialModelFileName :='New_Model/best_dl_model_detection.hdl'
else
    InitialModelFileName := ExampleDataDir + '/pretrained_dl_model_detection.hdl'
endif

DataDirectory := ExampleDataDir + '/dldataset'
DLDatasetFileName := DataDirectory + '/dl_dataset.hdict'
*保存至new_model文件夹
BestModelBaseName := 'New_Model/best_dl_model_detection'
FinalModelBaseName := 'New_Model/final_dl_model_detection'
ChangeLearningRateValues := InitialLearningRate * ChangeLearningRate
DisplayEvaluation := true
SeedRandom := 42

GenParamName := []
GenParamValue := []

if (|ChangeLearningRateEpochs| > 0)
    create_dict (ChangeStrategy)
    * Specify the model parameter to be changed, here the learning rate.
    set_dict_tuple (ChangeStrategy, 'model_param', 'learning_rate')
    * Start the parameter value at 'initial_value'.
    set_dict_tuple (ChangeStrategy, 'initial_value', InitialLearningRate)
    * Reduce the learning rate in the following epochs.
    set_dict_tuple (ChangeStrategy, 'epochs', ChangeLearningRateEpochs)
    * Reduce the learning rate to the following value at epoch 30.
    set_dict_tuple (ChangeStrategy, 'values', ChangeLearningRateValues)
    * Collect all change strategies as input.
    GenParamName := [GenParamName,'change']
    GenParamValue := [GenParamValue,ChangeStrategy]
endif

create_dict (SerializationStrategy)
set_dict_tuple (SerializationStrategy, 'type', 'best')
set_dict_tuple (SerializationStrategy, 'basename', BestModelBaseName)
GenParamName := [GenParamName,'serialize']
GenParamValue := [GenParamValue,SerializationStrategy]
create_dict (SerializationStrategy)
set_dict_tuple (SerializationStrategy, 'type', 'final')
set_dict_tuple (SerializationStrategy, 'basename', FinalModelBaseName)
GenParamName := [GenParamName,'serialize']
GenParamValue := [GenParamValue,SerializationStrategy]

SelectedPercentageTrainSamples := 20
* Set the x-axis argument of the training plots.
* XAxisLabel := 'epochs'
* create_dict (DisplayParam)
* set_dict_tuple (DisplayParam, 'selected_percentage_train_samples', SelectedPercentageTrainSamples)
* set_dict_tuple (DisplayParam, 'x_axis_label', XAxisLabel)
* GenParamName := [GenParamName,'display']
* GenParamValue := [GenParamValue,DisplayParam]

* Check if all necessary files exist.
check_data_availability (ExampleDataDir, InitialModelFileName, DLDatasetFileName)
 
* Read in the model that was initialized during preprocessing.
read_dl_model (InitialModelFileName, DLModelHandle)

* Read in the preprocessed DLDataset file.
read_dict (DLDatasetFileName, [], [], DLDataset)

* Set model hyper-parameters as specified in the settings above.
set_dl_model_param (DLModelHandle, 'learning_rate', InitialLearningRate)
set_dl_model_param (DLModelHandle, 'momentum', Momentum)

set_dl_model_param (DLModelHandle, 'batch_size', BatchSize)

if (|WeightPrior| > 0)
    set_dl_model_param (DLModelHandle, 'weight_prior', WeightPrior)
endif

ifelse (runtime_type == 'cpu')
    get_system ('version', version)
    ifelse (number(version) > 19.11)
        set_dl_model_param (DLModelHandle, 'runtime', runtime_type)
    else
        throw (['19.11及以下版本不支持cpu训练'])
    endif
else
    set_dl_model_param (DLModelHandle, 'runtime', runtime_type)
endif
create_dl_train_param (DLModelHandle, NumEpochs, EvaluationIntervalEpochs, DisplayEvaluation, SeedRandom, GenParamName, GenParamValue, TrainParam)
*halcon训练部分
train_dl_model (DLDataset, DLModelHandle, TrainParam, 0.0, TrainResults, TrainInfos, EvaluationInfos)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
return ()

3、模型评估：（Data_Evaluate_Rectangle1 ）:

dev_update_off ()

ExampleDataDir := 'detect_data'
DataDirectory := ExampleDataDir + '/dldataset'
DLDatasetFileName := DataDirectory + '/dl_dataset.hdict'

RetrainedModelFileName := 'New_Model/best_dl_model_detection.hdl'
* Specify measures of interest
EvaluationMeasures := 'all'
 
* Specify considered IoU thresholds.
IoUThresholds := []

* Specify evaluation subsets for objects of a certain size.
AreaNames := []
AreaMin := []
AreaMax := []

* Specify the maximum number of detections considered for each measure.
MaxNumDetections := []

* Check availability of GPU mode.
if (UseGPU)
    get_system ('cuda_loaded', CudaLoaded)
    get_system ('cudnn_loaded', CuDNNLoaded)
    get_system ('cublas_loaded', CuBlasLoaded)
    if (not (CudaLoaded == 'true' and CuDNNLoaded == 'true' and CuBlasLoaded == 'true'))
        UseGPU := false
    endif
endif

* Read the trained model.
read_dl_model (RetrainedModelFileName, DLModelHandle)

* Initialize the model.
if (not UseGPU)
    set_dl_model_param (DLModelHandle, 'runtime', 'cpu')
endif
* Set batch size of the model to 1 temporarily.
set_dl_model_param (DLModelHandle, 'batch_size', BatchSize)
set_dl_model_param (DLModelHandle, 'runtime_init', 'immediately')

* Read the evaluation data.
read_dict (DLDatasetFileName, [], [], DLDataset)

* To reduce the number of false positives, set lower values for
* 'max_overlap' (default = 0.5) and 'max_overlap_class_agnostic'
* (default = 1.0) and a higher confidence threshold (default = 0.5).
set_dl_model_param (DLModelHandle, 'max_overlap_class_agnostic', max_overlap_class_agnostic)
set_dl_model_param (DLModelHandle, 'max_overlap', max_overlap)
set_dl_model_param (DLModelHandle, 'min_confidence', min_confidence)

* Create parameter dictionaries for visualization.
create_dict (WindowHandleDict)
create_dict (GenParam)
set_dict_tuple (GenParam, 'bbox_display_confidence', false)
* 
* Select test images randomly.
get_dict_tuple (DLDataset, 'samples', DLSamples)
find_dl_samples (DLSamples, 'split', 'test', 'or', DLSampleIndices)
tuple_shuffle (DLSampleIndices, DLSampleIndicesShuffled)

set_dl_model_param (DLModelHandle, 'batch_size', BatchSize)

* Set generic evaluation parameters.
create_dict (GenParamEval)
* Set the measures of interest.
set_dict_tuple (GenParamEval, 'measures', EvaluationMeasures)
* Set maximum number of detections considered for each measure.
if (|MaxNumDetections|)
    set_dict_tuple (GenParamEval, 'max_num_detections', MaxNumDetections)
endif
* Set the evaluation area subsets.
if (|AreaNames|)
    if ((|AreaNames| != |AreaMin|) or (|AreaNames| != |AreaMax|))
        throw ('AreaNames, AreaMin, and AreaMax must have the same size.')
    endif
    create_dict (AreaRanges)
    set_dict_tuple (AreaRanges, 'name', AreaNames)
    set_dict_tuple (AreaRanges, 'min', AreaMin)
    set_dict_tuple (AreaRanges, 'max', AreaMax)
    set_dict_tuple (GenParamEval, 'area_ranges', AreaRanges)
endif
* Set IoU thresholds.
if (|IoUThresholds|)
    set_dict_tuple (GenParamEval, 'iou_threshold', IoUThresholds)
endif
* Enable detailed evaluation.
set_dict_tuple (GenParamEval, 'detailed_evaluation', true)
* Show progress of evaluation.
set_dict_tuple (GenParamEval, 'show_progress', true)
* 
* Evaluate the finetuned model on the 'test' split of the dataset.
evaluate_dl_model (DLDataset, DLModelHandle, 'split', 'test', GenParamEval, EvaluationResultDetection, EvalParams)
* 
* Display results of the detailed evaluation.
create_dict (DisplayParam)
* Set the IoU of interest. The default is the first 'iou_threshold' of EvalParams.
if (|DisplayIoUThreshold| == 1)
    get_dict_tuple (EvalParams, 'iou_threshold', EvalIoUThresholds)
    if (find(EvalIoUThresholds,DisplayIoUThreshold) != -1)
        set_dict_tuple (DisplayParam, 'iou_threshold', DisplayIoUThreshold)
    else
        throw ('No evaluation result for specified IoU threshold.')
    endif
endif
* Display detailed precision and recall
set_dict_tuple (DisplayParam, 'display_mode', ['pie_charts_precision','pie_charts_recall'])
create_dict (WindowHandleDict)
dev_display_detection_detailed_evaluation (EvaluationResultDetection, EvalParams, DisplayParam, WindowHandleDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'top', 'right', 'black', [], [])
stop ()
dev_close_window_dict (WindowHandleDict)
* Display confusion matrix.
set_dict_tuple (DisplayParam, 'display_mode', 'absolute_confusion_matrix')
dev_display_detection_detailed_evaluation (EvaluationResultDetection, EvalParams, DisplayParam, WindowHandleDict)
dev_disp_text ('Press Run (F5) to continue', 'window', 'bottom', 'right', 'black', [], [])
stop ()
dev_close_window_dict (WindowHandleDict)

return ()

4、推理初始化：（Data_Infer_Init_Rectangle1 ）:

dev_update_off ()
*预处理前删除上次数据
* file_exists ('detect_data', FileExists)
* if (FileExists)
*     remove_dir_recursively ('detect_data')
* endif

if (UseGPU)
    get_system ('cuda_loaded', CudaLoaded)
    get_system ('cudnn_loaded', CuDNNLoaded)
    get_system ('cublas_loaded', CuBlasLoaded)
    if (not (CudaLoaded == 'true' and CuDNNLoaded == 'true' and CuBlasLoaded == 'true'))
        UseGPU := false
    endif
endif

*'New_Model/best_dl_model_detection.hdl'
read_dl_model ('New_Model/best_dl_model_detection.hdl', DLModelHandle)
set_dl_model_param (DLModelHandle, 'batch_size', BatchSizeInference) 

if (not UseGPU)
    set_dl_model_param (DLModelHandle, 'runtime', 'cpu')
endif
set_dl_model_param (DLModelHandle, 'runtime_init', 'immediately')

set_dl_model_param (DLModelHandle, 'min_confidence', MinConfidence)
set_dl_model_param (DLModelHandle, 'max_overlap', MaxOverlap)
set_dl_model_param (DLModelHandle, 'max_overlap_class_agnostic', MaxOverlapClassAgnostic)
read_dict ('New_Model/dl_preprocess_param.hdict', [], [], DLPreprocessParam)

return ()

5、在线部署运行：（Data_Infer_OnLine_Rectangle1 ）:

get_dict_tuple (DictHandle,'class_names' , class_names)

*模型的宽和高
get_dict_tuple (DLPreprocessParam, 'image_width', Model_Width)
get_dict_tuple (DLPreprocessParam, 'image_height', Model_Height)

gen_dl_samples_from_images (Image, DLSampleBatch)
preprocess_dl_samples (DLSampleBatch, DLPreprocessParam)
apply_dl_model (DLModelHandle, DLSampleBatch, [], DLResultBatch)

get_dict_tuple (DLResultBatch, 'bbox_row1', bbox_row1)
get_dict_tuple (DLResultBatch, 'bbox_col1', bbox_col1)
get_dict_tuple (DLResultBatch, 'bbox_row2', bbox_row2)
get_dict_tuple (DLResultBatch, 'bbox_col2', bbox_col2)

get_dict_tuple (DLResultBatch, 'bbox_class_id', bbox_class_id)

get_dict_tuple (DLResultBatch, 'bbox_confidence', bbox_confidence)

get_image_size (Image, Width, Height)

bbox_row1 := bbox_row1 * (Height * 1.0/ Model_Height * 1.0)
bbox_row2 := bbox_row2 * (Height * 1.0/ Model_Height * 1.0)

bbox_col1 := bbox_col1 * (Width * 1.0/ Model_Width * 1.0)
bbox_col2 := bbox_col2 * (Width * 1.0/ Model_Width * 1.0)

* if (|bbox_class_id|)
*     gen_rectangle1 (Rectangle, return_bbox_row1, return_bbox_col1, return_bbox_row2, return_bbox_col2)
*     dev_clear_window ()
*     dev_display (Image)
*     boundary (Rectangle, Rectangle, 'inner')
*     dev_disp_text (class_names[bbox_class_id], 'image', return_bbox_row1, return_bbox_col1, 'red', 'box', 'false')
* endif
return ()