Hierarchy Multi-Class label Classification using LSTM

Hi, In this blog, I am going to explain shortly about the multi-class label classification using lstm and also I am going to explain in which scenories lstm going to helpful as out in multi-class label classification.

Let say , we have a vehicle identification dataset, In this dataset we having vehicle type like bus,car,bike etc., then, vehicle brand type like volvo,hyandi, ktm etc., and final we have vehicle model type of each brand such as volvo delexe , hyandi i20, ktm 200, ktm 790 , ktm adventure. As we can see that each label is maintaining a hierarchy order , So that instead of using direct connect between the convolution layer and also dense layer, we going to use lstm so that we can use this hierarchy order to train the lstm model.

Why we are using lstm for hierarchical multi label?

In multi-class label classification normally we can train it in two ways. but if train on both way we are breaking the corelation or hierarchy between the label or in other words multiple labels train as a independent to each other.

Case 1:

In first case ,If we train the model as single dense layer, by making the output as a multi-hot encoder,we are breaking the hierarchy but, we can say that initiatively somewhat corelation is maintained between the label, but we can’t guarantee that the model correlated the each and every label properly.

One more disadvantage of this process is we can’t use the softmax activation function.So that it make the convergences more difficult in my point of view.

Case 2:

In second case ,If we train the model with multi dense layer, by making the output as a one-hot encoder,we are breaking the hierarchy and also corelation. But this model convergence much faster then the first and also we can use softmax activation function.

So to solve this hierarchy problem , as I said we are going to use lstm, So that in first time step vehicle type is predicted then, next step we predict the brand according to the first predict like that each and every prediction depends on the previous predict.

Note: This lstm classification will give has benefit ,only when the multi label has a corelation and also dependence on each other in a hierarchical manner.

Sample Code:

import math
import os
import sys
import argparse
import matplotlib
import imghdr
import pickle as pkl
import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
from keras.applications.xception import Xception, preprocess_input
from tqdm.keras import TqdmCallback
# from keras.applications.mobilenet_v2 import preprocess_input,MobileNetV2
from keras.optimizers import Adam,Nadam
from keras.preprocessing import image
from keras.losses import categorical_crossentropy,binary_crossentropy,hinge
from keras.layers import Dense, GlobalAveragePooling2D,RepeatVector,Bidirectional,LSTM,TimeDistributed,Dropout
from keras.models import Model
from keras.utils import to_categorical
from keras.callbacks import ModelCheckpoint,EarlyStopping,ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import LeakyReLU
from keras.models import model_from_json
from tqdm import tqdm
import cv2
import pathlib
# import lodgepole.image_tools as lit
import pandas as pd
LOG_LEVEL_INDEX = sys.argv.index('--log_level') + 1 if '--log_level' in sys.argv else 0
os.environ['TF_CPP_MIN_LOG_LEVEL'] = sys.argv[LOG_LEVEL_INDEX] if 0 < LOG_LEVEL_INDEX < len(sys.argv) else '3'
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
os.environ["CUDA_VISIBLE_DEVICES"]="0"
matplotlib.use('Agg')
current_directory = os.path.dirname(os.path.abspath(__file__))
parser = argparse.ArgumentParser()
# parser.add_argument('dataset_root')
# parser.add_argument('classes')
# parser.add_argument('result_root')
parser.add_argument('--epochs_pre', type=int, default=100)
parser.add_argument('--epochs_fine', type=int, default=200)
parser.add_argument('--batch_size_pre', type=int, default=4)
parser.add_argument('--batch_size_fine', type=int, default=2)
parser.add_argument('--lr_pre', type=float, default=1e-3)
parser.add_argument('--lr_fine', type=float, default=1e-4)
parser.add_argument('--snapshot_period_pre', type=int, default=1)
parser.add_argument('--snapshot_period_fine', type=int, default=1)
parser.add_argument('--split', type=float, default=0.9)
# with open('classes_vehicle_classification_pruned.txt','r') as f:
#     columns = f.read().split('\n')
# with open('view.txt','r') as f:
#     columns_view = f.read().split('\n')
# print(columns)
# def gray_to_rgb(img):
#     return np.repeat(img, 3, 2)
# def grayConversion(image):
#     grayValue = 0.07 * image[:,:,2] + 0.72 * image[:,:,1] + 0.21 * image[:,:,0]
#     gray_img = grayValue.astype(np.uint8)
#     # gray_img = np.repeat(gray_img, 3, 2)
#     return gray_img
def generate_from_paths_and_labels(generator,input_paths, labels_brand,labels_view,labels_model, batch_size, input_size=(299,299)):
num_samples = len(input_paths)
print(num_samples,':::::::::::::::::::')
#     with open('testing_path.txt','a+') as f:
#         f.write('\n'.join(input_paths))
while 1:
perm = np.random.permutation(num_samples)
input_paths = input_paths[perm]
labels_brand = labels_brand[perm]
labels_view = labels_view[perm]
labels_model = labels_model[perm]
#         print('-----------------------------start-------------------------------')
#         print(labels_brand.shape,'::::::::::::::::::::::::::<<<<<<<<<<<<<<<<<<<<<',labels_model.shape)
# labels_brand_2 = np.pad(labels_brand,((0,0),(0,len(columns_model))),'constant').astype('int8')
# labels_model_2 = np.pad(labels_model,((0,0),(len(columns_brand),0)), 'constant').astype('int8')
# print(labels_brand_2.shape,'::::::::::::::::::::::::::<<<<<<<<<<<<<<<<<<<<<',labels_model_2.shape)
# labels_final = np.stack((np.pad(labels_brand,((0,0),(0,len(columns_model))),'constant'), np.pad(labels_model,((0,0),(len(columns_brand),0)), 'constant'),axis=1))
labels_final = np.stack((np.pad(labels_brand,((0,0),(0,len(columns_model))),'constant').astype('int8'), np.pad(labels_model,((0,0),(len(columns_brand),0)), 'constant').astype('int8')),axis=1).astype('int8')
#         print('------------------------------end-----------------------------------')
#         print(labels_view.shape)
for i in range(0, num_samples, batch_size):
# previous = ''
# inputs = []
# for x in input_paths[i:i+batch_size]:
#     try:
#         inputs.append(image.load_img(x, target_size=input_size))
#         previous = x
#     except:
#         inputs.append(image.load_img(previous, target_size=input_size))
inputs = list(map(
lambda x: image.load_img(x, target_size=input_size),
input_paths[i:i+batch_size]
))
inputs = np.array(list(map(
lambda x: image.img_to_array(x),
inputs
)))
# inputs = preprocess_input(inputs)
inputs = list(map(
lambda x: next(generator.flow(x[np.newaxis],shuffle=False,batch_size=1)),#image.load_img(x, target_size=input_size),
inputs
))
inputs = list(map(
lambda x: x.reshape(299,299,3),#image.load_img(x, target_size=input_size),
inputs
))
# inputs = np.array(inputs)
# inputs = list(map(
#     lambda x: np.stack((lit.rgb2gray_approx(x),)*3,axis=-1),#image.load_img(x, target_size=input_size),
#     inputs
# ))
# stacked_img = np.stack((img,)*3, axis=-1)
# print(input[0].shape)
# inputs = list(map(
#     lambda x: x.reshape(299,299,3),#image.load_img(x, target_size=input_size),
#     inputs
# ))
inputs = np.array(inputs).astype('float32')
# inputs
# inputs = np.array(list(map(
#     lambda x: image.img_to_array(x),
#     inputs
# )))
#             inputs = preprocess_input(inputs)
#             inputs = inputs.astype('float32')
#             yield (inputs, {'dense':labels_view[i:i+batch_size]})
#             print(labels_final.shape,'::::::::::::::::::::::>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<')
yield (inputs, {'time_distributed_1':labels_final[i:i+batch_size],'dense_1':labels_view[i:i+batch_size]})
# def generate_from_paths_and_labels_no_augmentation(input_paths, labels_brand,labels_view, batch_size, input_size=(299,299)):
#     num_samples = len(input_paths)
#     while 1:
#         perm = np.random.permutation(num_samples)
#         input_paths = input_paths[perm]
#         labels_brand = labels_brand[perm]
#         labels_view = labels_view[perm]
#         for i in range(0, num_samples, batch_size):
#             previous = ''
#             inputs = []
#             for x in input_paths[i:i+batch_size]:
#                 try:
#                     inputs.append(image.load_img(x, target_size=input_size))
#                     previous = x
#                 except:
#                     inputs.append(image.load_img(previous, target_size=input_size))
#             # inputs = list(map(
#             #     lambda x: image.load_img(x, target_size=input_size),
#             #     input_paths[i:i+batch_size]
#             # ))
#             inputs = np.array(list(map(
#                 lambda x: image.img_to_array(x),
#                 inputs
#             )))
#             inputs = list(map(
#                 lambda x: x.reshape(299,299,3),#image.load_img(x, target_size=input_size),
#                 inputs
#             ))
#             # inputs = np.array(inputs)
#             # inputs = list(map(
#             #     lambda x: np.stack((lit.rgb2gray_approx(x),)*3,axis=-1),#image.load_img(x, target_size=input_size),
#             #     inputs
#             # ))
#             inputs = np.array(inputs)
#             # inputs = list(map(
#             #     lambda x: x.reshape(299,299,3),#image.load_img(x, target_size=input_size),
#             #     inputs
#             # ))
#             # inputs = preprocess_input(inputs)
#             # inputs = list(map(
#             #     lambda x: next(generator.flow(x[np.newaxis],shuffle=False,batch_size=1)),#image.load_img(x, target_size=input_size),
#             #     inputs
#             # ))
#             # inputs = list(map(
#             #     lambda x: x.reshape(299,299,3),#image.load_img(x, target_size=input_size),
#             #     inputs
#             # ))
#             # inputs = np.array(inputs)
#             # inputs = np.array(list(map(
#             #     lambda x: image.img_to_array(x),
#             #     inputs
#             # )))
#             inputs = preprocess_input(inputs)
#             yield (inputs, {'output_final':labels_final[i:i+batch_size],'output_view':labels_view[i:i+batch_size]})
import tensorflow as tf
K = tf.keras.backend
def binary_focal_loss(gamma=2., alpha=.25):
"""
Binary form of focal loss.
FL(p_t) = -alpha * (1 - p_t)**gamma * log(p_t)
where p = sigmoid(x), p_t = p or 1 - p depending on if the label is 1 or 0, respectively.
References:
https://arxiv.org/pdf/1708.02002.pdf
Usage:
model.compile(loss=[binary_focal_loss(alpha=.25, gamma=2)], metrics=["accuracy"], optimizer=adam)
"""
def binary_focal_loss_fixed(y_true, y_pred):
"""
:param y_true: A tensor of the same shape as `y_pred`
:param y_pred:  A tensor resulting from a sigmoid
:return: Output tensor.
"""
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
epsilon = K.epsilon()
# clip to prevent NaN's and Inf's
pt_1 = K.clip(pt_1, epsilon, 1. - epsilon)
pt_0 = K.clip(pt_0, epsilon, 1. - epsilon)
return -K.sum(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) \
-K.sum((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))
return binary_focal_loss_fixed
def categorical_focal_loss(gamma=2., alpha=.25):
"""
Softmax version of focal loss.
m
FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)
c=1
where m = number of classes, c = class and o = observation
Parameters:
alpha -- the same as weighing factor in balanced cross entropy
gamma -- focusing parameter for modulating factor (1-p)
Default value:
gamma -- 2.0 as mentioned in the paper
alpha -- 0.25 as mentioned in the paper
References:
Official paper: https://arxiv.org/pdf/1708.02002.pdf
https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy
Usage:
model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=["accuracy"], optimizer=adam)
"""
def categorical_focal_loss_fixed(y_true, y_pred):
"""
:param y_true: A tensor of the same shape as `y_pred`
:param y_pred: A tensor resulting from a softmax
:return: Output tensor.
"""
# Scale predictions so that the class probas of each sample sum to 1
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
# Clip the prediction value to prevent NaN's and Inf's
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
# Calculate Cross Entropy
cross_entropy = -y_true * K.log(y_pred)
# Calculate Focal Loss
loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy
# Sum the losses in mini_batch
return K.sum(loss, axis=1)
return categorical_focal_loss_fixed
# def my_gen(gen):
#     while True:
#         try:
#             data, labels = next(gen)
#             yield data, labels
#         except:
#             pass
with open('./bus/vehicle_type_brand_column_bus.txt','r') as f:
columns_brand = f.read().split('\n')
with open('./bus/vehicle_type_model_column_bus.txt','r') as f:
columns_model = f.read().split('\n')
def main(args):
with open('view.txt','r') as f:
columns_view = f.read().split('\n')
csv_path = list(glob.glob('./bus/*.csv'))[0]
model_name_save = csv_path.split('/')[-1]
model_name_save = model_name_save.replace('.csv','')
result_root = './separate_model_h5_file/'+csv_path.split('/')[-1].replace('.csv','')
pathlib.Path(result_root).mkdir(parents=True,exist_ok=True)
# ====================================================
# Preparation
# ====================================================
# parameters
# epochs = args.epochs_pre + args.epochs_fine
# args.dataset_root = os.path.expanduser(args.dataset_root)
# args.result_root = os.path.expanduser(args.result_root)
# args.classes = os.path.expanduser(args.classes)
# load class names
# with open(args.classes, 'r') as f:
#     classes = f.readlines()
#     classes = list(map(lambda x: x.strip(), classes))
num_classes = len(columns_brand)+ len(columns_model)
# if os.path.exists(args.result_root) is False:
#     os.makedirs(args.result_root)
# model_vehicle_pre_final.h5
# ====================================================
# Build a custom Xception
# ====================================================
# instantiate pre-trained Xception model
# the default input shape is (299, 299, 3)
# NOTE: the top classifier is not included
base_model = Xception(
include_top=False,
weights='imagenet',
input_shape=(299, 299, 3))
print('num_classes:',num_classes,'::::::::::::::::::::::::::::::>>>>>>>')
# create a custom top classifier
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024,activation='elu')(x)
x = Dropout(0.3)(x)
dense2 = Dense(len(columns_view), activation='softmax')(x)
x = RepeatVector(2)(x)
LSTM1 = Bidirectional(LSTM(64, activation='elu',return_sequences=True,dropout=0.2,recurrent_dropout=0.2))(x)
LSTM1 = Bidirectional(LSTM(32, activation='elu',return_sequences=True,dropout=0.2,recurrent_dropout=0.2))(LSTM1)
dense4 = TimeDistributed(Dense(1024, activation='elu'))(LSTM1)
dense1 = TimeDistributed(Dense(num_classes, activation='softmax'))(dense4)
# for i in range(num_classes):
#     layer_list.append(Dense(1, activation='sigmoid')(x))
model = Model(inputs=base_model.inputs, outputs=[dense1,dense2])
# model.load_weights('./separate_model_h5_file/vehicle_type_auto_training/model_vehicle_fine_ep42_valloss5.664.h5')
#         model = Model(inputs=base_model.inputs, outputs=dense1)
for layer in model.layers:
layer.trainable = True
# model.load_weights('./separate_model_h5_file/vehicle_type_car_training/model_vehicle_fine_ep1_valloss7.839.h5')
train_datagen = ImageDataGenerator(
#             rescale=1./255,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=[0.9, 1.25],
brightness_range=[0.5, 1.5],
rotation_range=20,
fill_mode='reflect',
horizontal_flip=True,)
df = pd.read_csv(csv_path)
labels_brand = df[columns_brand].values
labels_model =  df[columns_model].values
labels_view =  df[columns_view].values
labels_brand = labels_brand.astype('float32')
labels_model = labels_model.astype('float32')
labels_view = labels_view.astype('float32')
#         labels_brand = labels_brand.astype('float32')
# convert to numpy array
input_paths = df['path'].values
# shuffle dataset
perm = np.random.permutation(len(input_paths))
labels_brand = labels_brand[perm]
labels_model = labels_model[perm]
labels_view = labels_view[perm]
input_paths = input_paths[perm]
# split dataset for training and validation
border = int(len(input_paths) * args.split)
train_labels_brand, val_labels_brand = labels_brand, labels_brand[border:]
train_labels_model, val_labels_model = labels_model, labels_model[border:]
train_labels_view, val_labels_view = labels_view, labels_view[border:]
train_input_paths, val_input_paths = input_paths, input_paths[border:]
# train_labels_brand, val_labels_brand = labels_brand[:border], labels_brand[border:]
# train_labels_view, val_labels_view = labels_view[:border], labels_view[border:]
# train_input_paths, val_input_paths = input_paths[:border], input_paths[border:]
print("Training on %d images and labels" % (len(train_input_paths)))
print("Validation on %d images and labels" % (len(val_input_paths)))
STEP_SIZE_TRAIN = math.ceil(len(train_input_paths) / args.batch_size_pre)
STEP_SIZE_VALID = math.ceil(len(val_input_paths) / args.batch_size_pre)
model.compile(
optimizer=Nadam(lr=1e-4),
loss=[categorical_crossentropy,categorical_crossentropy],#[categorical_focal_loss(),categorical_focal_loss()],
#[categorical_crossentropy,categorical_crossentropy,categorical_crossentropy],
# loss_weights=[0.9,1.0],[categorical_focal_loss(alpha=.25, gamma=2),categorical_focal_loss(alpha=.25, gamma=2),categorical_focal_loss(alpha=.25, gamma=2)]
metrics=['accuracy'])
model.summary()
#     # train
#     STEP_SIZE_TRAIN = train_generator.samples // args.batch_size_fine
#     STEP_SIZE_VALID = validation_generator.samples // args.batch_size_fine
training_generator = generate_from_paths_and_labels(
generator = train_datagen,
input_paths=train_input_paths,
labels_view=train_labels_view,
labels_brand=train_labels_brand,
labels_model=train_labels_model,
batch_size=args.batch_size_pre
)
validation_generator = generate_from_paths_and_labels(
generator = train_datagen,
input_paths=val_input_paths,
labels_view=val_labels_view,
labels_brand=val_labels_brand,
labels_model=train_labels_model,
batch_size=args.batch_size_pre
)
hist_fine = model.fit_generator(generator=training_generator,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data =validation_generator,
validation_steps=STEP_SIZE_VALID,
verbose=1,
callbacks=[
ModelCheckpoint(
filepath=os.path.join(result_root, 'model_vehicle_fine_ep{epoch}_valloss{val_loss:.3f}.h5'),
period=args.snapshot_period_pre,
),
],
epochs=args.epochs_fine
)#, TqdmCallback(verbose=1)
'''EarlyStopping(monitor = 'val_loss',
min_delta = 0,
patience = 10,
verbose = 0,
restore_best_weights = True),
ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=3  )'''
model.save(os.path.join(result_root, 'model_vehicle_fine_final_'+model_name_save+'.h5'))
model_json = model.to_json()
with open(os.path.join(result_root,'model_vehicle_fine_final_'+model_name_save+'.json'), "w") as json_file:
json_file.write(model_json)
K.clear_session()
# ====================================================
# Create & save result graphs
# ====================================================
# concatinate plot data
# acc = hist_pre.history['accuracy']
# val_acc = hist_pre.history['val_accuracy']
# loss = hist_pre.history['loss']
# val_loss = hist_pre.history['val_loss']
# acc.extend(hist_fine.history['accuracy'])
# val_acc.extend(hist_fine.history['val_accuracy'])
# loss.extend(hist_fine.history['loss'])
# val_loss.extend(hist_fine.history['val_loss'])
# # save graph image
# plt.plot(range(epochs), acc, marker='.', label='accuracy')
# plt.plot(range(epochs), val_acc, marker='.', label='val_accuracy')
# plt.legend(loc='best')
# plt.grid()
# plt.xlabel('epoch')
# plt.ylabel('accuracy')
# plt.savefig(os.path.join(args.result_root, 'accuracy.png'))
# plt.clf()
# plt.plot(range(epochs), loss, marker='.', label='loss')
# plt.plot(range(epochs), val_loss, marker='.', label='val_loss')
# plt.legend(loc='best')
# plt.grid()
# plt.xlabel('epoch')
# plt.ylabel('loss')
# plt.savefig(os.path.join(args.result_root, 'loss.png'))
# plt.clf()
# # save plot data as pickle file
# plot = {
#     'accuracy': acc,
#     'val_accuracy': val_acc,
#     'loss': loss,
#     'val_loss': val_loss,
# }
# with open(os.path.join(args.result_root, 'plot.dump'), 'wb') as f:
#     pkl.dump(plot, f)
if __name__ == '__main__':
args = parser.parse_args()
main(args)

Inference Code:

import argparse
import numpy as np
from keras.applications.xception import preprocess_input
from keras.preprocessing import image
from keras.models import load_model
import os
parser = argparse.ArgumentParser()
parser.add_argument('model')
parser.add_argument('classes_brand')
parser.add_argument('classes_model')
parser.add_argument('classes_view')
parser.add_argument('image')
parser.add_argument('--top_n', type=int, default=1)
import tensorflow as tf
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
os.environ["CUDA_VISIBLE_DEVICES"]="0"
K = tf.keras.backend
def categorical_focal_loss(gamma=2., alpha=.25):
"""
Softmax version of focal loss.
m
FL = ∑  -alpha * (1 - p_o,c)^gamma * y_o,c * log(p_o,c)
c=1
where m = number of classes, c = class and o = observation
Parameters:
alpha -- the same as weighing factor in balanced cross entropy
gamma -- focusing parameter for modulating factor (1-p)
Default value:
gamma -- 2.0 as mentioned in the paper
alpha -- 0.25 as mentioned in the paper
References:
Official paper: https://arxiv.org/pdf/1708.02002.pdf
https://www.tensorflow.org/api_docs/python/tf/keras/backend/categorical_crossentropy
Usage:
model.compile(loss=[categorical_focal_loss(alpha=.25, gamma=2)], metrics=["accuracy"], optimizer=adam)
"""
def categorical_focal_loss_fixed(y_true, y_pred):
"""
:param y_true: A tensor of the same shape as `y_pred`
:param y_pred: A tensor resulting from a softmax
:return: Output tensor.
"""
# Scale predictions so that the class probas of each sample sum to 1
y_pred /= K.sum(y_pred, axis=-1, keepdims=True)
# Clip the prediction value to prevent NaN's and Inf's
epsilon = K.epsilon()
y_pred = K.clip(y_pred, epsilon, 1. - epsilon)
# Calculate Cross Entropy
cross_entropy = -y_true * K.log(y_pred)
# Calculate Focal Loss
loss = alpha * K.pow(1 - y_pred, gamma) * cross_entropy
# Sum the losses in mini_batch
return K.sum(loss, axis=1)
return categorical_focal_loss_fixed
def main(args):
# create model
model = load_model(args.model,custom_objects={'categorical_focal_loss_fixed':categorical_focal_loss()})
#     model = load_model(args.model,)
# load class names
classes = []
with open(args.classes_brand, 'r') as f:
classes_brand = list(map(lambda x: x.strip(), f.readlines()))
with open(args.classes_model, 'r') as f:
classes_model = list(map(lambda x: x.strip(), f.readlines()))
with open(args.classes_view, 'r') as f:
classes_view = list(map(lambda x: x.strip(), f.readlines()))
classes_brand = classes_brand + classes_model
print(classes_brand)
print(classes_model)
print(classes_view)
# load an input image
img = image.load_img(args.image, target_size=(299, 299))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
#     x = x/255.0
#     x = preprocess_input(x)
#     print(x)
#     x = x.astype('float32')
# predict
pred = model.predict(x)
pred_view = pred[1][0]
pred_brand = pred[0][0][0]
pred_model = pred[0][0][1]
print(pred_model.shape,'::::::::::')
result_view = [(classes_view[i], float(pred_view[i]) * 100.0) for i in range(len(pred_view))]
result_view.sort(reverse=True, key=lambda x: x[1])
result_brand = [(classes_brand[i], float(pred_brand[i]) * 100.0) for i in range(len(pred_brand))]
result_brand.sort(reverse=True, key=lambda x: x[1])
result_model = [(classes_brand[i], float(pred_model[i]) * 100.0) for i in range(len(pred_model))]
result_model.sort(reverse=True, key=lambda x: x[1])
for i in range(args.top_n):
(class_name, prob) = result_view[i]
print("Top %d ====================" % (i + 1))
print("Class name: %s" % (class_name))
print("Probability: %.2f%%" % (prob))
(class_name, prob) = result_brand[i]
print("Top %d ====================" % (i + 1))
print("Class name_brand: %s" % (class_name))
print("Probability: %.2f%%" % (prob))
(class_name, prob) = result_model[i]
print("Top %d ====================" % (i + 1))
print("Class name_model: %s" % (class_name))
print("Probability: %.2f%%" % (prob))
if __name__ == '__main__':
args = parser.parse_args()
main(args)

Sample dataset:

csv file should contain one hot encoder for every label

then text file which contain the label

path , brand_name, model_name

./x.jpg 0 1 0

pathashokleylandbharatbenzeicherforcemotormotormahindrascaniasissirasmlisuzuswarajmazda…tataultraelectricevtatawingerplatinumtruckunknownvolvo.1volvocitybusvolvohybridcitybusback_viewfront_viewside_view0/mnt/media/vehicle_identification/vehicle_dete…000000010…00000000101icle_dete…000000010…00000000102/vehicle_identification/vehicle_dete…000000010…00000000103/vehicle_identification/vehicle_dete…000000010…00000000104vehicle_identification/vehicle_dete…000000010…0000000010…………………………………………………………484icle_dete…000100000…0000000010485vehicle_identification/vehicle_dete…000100000…0000000010486/vehicle_identification/vehicle_dete…000100000…0000000010487vehicle_identification/vehicle_dete…000100000…0000000100488vehicle_identification/vehicle_dete…000100000…0000000010

Thanks for reading…………………..