AI object detection via yolov5 didn't work out too well, in fact it was
crashing the detection threads for whatever reason. I could deep dive
why it was crashing but I think the better solution is to bring back
optical flow detection at the block level. the advantage of this over
object detection is the fact that a block doesn't need to have a whole
object in it.
This commit is contained in:
Maurice ONeal 2022-10-04 18:12:32 -04:00
parent 42814ab8ca
commit 4bf3672a39
10 changed files with 69 additions and 211 deletions

View File

@ -1,80 +0,0 @@
person
bicycle
car
motorbike
aeroplane
bus
train
truck
boat
traffic light
fire hydrant
stop sign
parking meter
bench
bird
cat
dog
horse
sheep
cow
elephant
bear
zebra
giraffe
backpack
umbrella
handbag
tie
suitcase
frisbee
skis
snowboard
sports ball
kite
baseball bat
baseball glove
skateboard
surfboard
tennis racket
bottle
wine glass
cup
fork
knife
spoon
bowl
banana
apple
sandwich
orange
broccoli
carrot
hot dog
pizza
donut
cake
chair
sofa
pottedplant
bed
diningtable
toilet
tvmonitor
laptop
mouse
remote
keyboard
cell phone
microwave
oven
toaster
sink
refrigerator
book
clock
vase
scissors
teddy bear
hair drier
toothbrush

Binary file not shown.

View File

@ -1,6 +1,3 @@
#!/bin/sh #!/bin/sh
mkdir -p /etc/mow
cp ./etc/yolov5s.onnx /etc/mow/yolov5s.onnx
cp ./etc/classes.txt /etc/mow/classes.txt
cp ./.build-mow/mow /usr/bin/mow cp ./.build-mow/mow /usr/bin/mow

View File

@ -140,18 +140,6 @@ string genDstFile(const string &dirOut, const char *fmt, const string &ext)
return cleanDir(dirOut) + string("/") + genTimeStr(fmt) + ext; return cleanDir(dirOut) + string("/") + genTimeStr(fmt) + ext;
} }
string genTmpFile(const string &dirOut, const string &ext, shared_t *share)
{
createDirTree(cleanDir(dirOut));
if (share->tmpId == 9999999)
{
share->tmpId = 0;
}
return cleanDir(dirOut) + string("/") + to_string(share->tmpId++) + ext;
}
Mat toGray(const Mat &src) Mat toGray(const Mat &src)
{ {
Mat ret; Mat ret;
@ -181,21 +169,6 @@ void rdLine(const string &param, const string &line, int *value)
} }
} }
vector<string> loadClassList()
{
vector<string> ret;
ifstream ifs("/etc/mow/classes.txt");
string line;
while (getline(ifs, line))
{
ret.push_back(line);
}
return ret;
}
bool rdConf(shared_t *share) bool rdConf(shared_t *share)
{ {
ifstream varFile(share->conf.c_str()); ifstream varFile(share->conf.c_str());
@ -224,11 +197,6 @@ bool rdConf(shared_t *share)
share->vidExt = "mp4"; share->vidExt = "mp4";
share->recLoopWait = false; share->recLoopWait = false;
share->skipCmd = false; share->skipCmd = false;
share->network = dnn::readNet("/etc/mow/yolov5s.onnx");
share->classNames = loadClassList();
share->network.setPreferableBackend(dnn::DNN_BACKEND_OPENCV);
share->network.setPreferableTarget(dnn::DNN_TARGET_CPU);
do do
{ {
@ -266,6 +234,9 @@ bool rdConf(shared_t *share)
new thread(enforceMaxDays, share); new thread(enforceMaxDays, share);
createDirTree(cleanDir(share->buffDir));
system(string("touch " + cleanDir(share->buffDir) + "/stat").c_str());
share->retCode = 0; share->retCode = 0;
} }
@ -336,7 +307,7 @@ string parseForParam(const string &arg, int argc, char** argv, bool argOnly)
void statOut(shared_t *share) void statOut(shared_t *share)
{ {
createDirTree(cleanDir(share->buffDir)); system(string("touch " + cleanDir(share->buffDir) + "/stat").c_str());
auto path = string(cleanDir(share->buffDir) + "/stat"); auto path = string(cleanDir(share->buffDir) + "/stat");
auto fd = open(path.c_str(), O_WRONLY); auto fd = open(path.c_str(), O_WRONLY);

View File

@ -37,26 +37,20 @@ using namespace std;
using namespace std::filesystem; using namespace std::filesystem;
#define BUF_SZ 10 #define BUF_SZ 10
#define APP_VER "1.4.t7" #define APP_VER "1.4.t8"
struct shared_t struct shared_t
{ {
vector<string> classNames;
dnn::Net network;
string stat; string stat;
string recordUrl; string recordUrl;
string outDir; string outDir;
string postCmd; string postCmd;
string conf; string conf;
string buffDir; string buffDir;
string concatTxtTmp;
string concatShTmp;
string createShTmp;
string vidExt; string vidExt;
bool init; bool init;
bool recLoopWait; bool recLoopWait;
bool skipCmd; bool skipCmd;
int tmpId;
int colorThresh; int colorThresh;
int secs; int secs;
int blockThresh; int blockThresh;
@ -66,7 +60,6 @@ struct shared_t
int retCode; int retCode;
}; };
string genTmpFile(const string &dirOut, const string &ext, shared_t *share);
string genDstFile(const string &dirOut, const char *fmt, const string &ext); string genDstFile(const string &dirOut, const char *fmt, const string &ext);
string genTimeStr(const char *fmt); string genTimeStr(const char *fmt);
string cleanDir(const string &path); string cleanDir(const string &path);
@ -84,6 +77,5 @@ bool rdConf(shared_t *share);
bool capPair(Mat &prev, Mat &next, VideoCapture &capture, shared_t *share); bool capPair(Mat &prev, Mat &next, VideoCapture &capture, shared_t *share);
Mat toGray(const Mat &src); Mat toGray(const Mat &src);
vector<string> lsFilesInDir(const string &path, const string &ext); vector<string> lsFilesInDir(const string &path, const string &ext);
vector<string> loadClassList();
#endif // COMMON_H #endif // COMMON_H

View File

@ -11,7 +11,6 @@
// GNU General Public License for more details. // GNU General Public License for more details.
#include "mo_detect.h" #include "mo_detect.h"
#include "obj_detect.h"
void detectLoop(shared_t *share) void detectLoop(shared_t *share)
{ {
@ -23,16 +22,11 @@ void detectLoop(shared_t *share)
if ((bufFiles.size() >= 2) || (share->recLoopWait && !bufFiles.empty())) if ((bufFiles.size() >= 2) || (share->recLoopWait && !bufFiles.empty()))
{ {
Rect blockArea;
Mat blockImg;
auto fullPath = cleanDir(share->buffDir) + "/" + bufFiles[0]; auto fullPath = cleanDir(share->buffDir) + "/" + bufFiles[0];
share->stat.clear(); share->stat.clear();
if (moDetect(fullPath, &blockArea, &blockImg, share)) if (moDetect(fullPath, share))
{
if (objectInImage(blockImg, blockArea, share))
{ {
share->skipCmd = true; share->skipCmd = true;
@ -42,11 +36,6 @@ void detectLoop(shared_t *share)
{ {
remove(fullPath.c_str()); remove(fullPath.c_str());
} }
}
else
{
remove(fullPath.c_str());
}
statOut(share); statOut(share);
} }
@ -108,7 +97,6 @@ int main(int argc, char** argv)
else else
{ {
sharedRes.retCode = 0; sharedRes.retCode = 0;
sharedRes.tmpId = 0;
sharedRes.recLoopWait = false; sharedRes.recLoopWait = false;
sharedRes.skipCmd = false; sharedRes.skipCmd = false;
sharedRes.init = true; sharedRes.init = true;

View File

@ -58,10 +58,8 @@ void secDiff(const Mat &imgA, const Mat &imgB, int rows, int cols, int rowOffs,
results->push_back(res); results->push_back(res);
} }
bool imgDiff(const Mat &prev, const Mat &next, Rect *block, shared_t *share) bool imgDiff(const Mat &prev, const Mat &next, shared_t *share)
{ {
auto moInBlock = false;
vector<thread> threads; vector<thread> threads;
vector<sec_t> results; vector<sec_t> results;
mutex secMutex; mutex secMutex;
@ -107,23 +105,20 @@ bool imgDiff(const Mat &prev, const Mat &next, Rect *block, shared_t *share)
if (maxPixDiff >= share->blockThresh) if (maxPixDiff >= share->blockThresh)
{ {
// return true on this function with the block with the // run optical flow calcuations on the block with the highest
// high pixDiff value which should be at or exceeds // pixDiff.
// the block_threshold set by the conf file.
auto res = results[blockPick]; auto res = results[blockPick];
auto block = Rect(res.x, res.y, res.xSize, res.ySize);
block->x = res.x; return objectFlowInImage(prev, next, block, share);
block->y = res.y; }
block->height = res.ySize; else
block->width = res.xSize; {
return false;
moInBlock = true; }
} }
return moInBlock; bool moDetect(const string &buffFile, shared_t *share)
}
bool moDetect(const string &buffFile, Rect *block, Mat *img, shared_t *share)
{ {
auto mod = false; auto mod = false;
@ -136,13 +131,10 @@ bool moDetect(const string &buffFile, Rect *block, Mat *img, shared_t *share)
share->stat += "motion_detection-- clip_file - " + buffFile + "\n"; share->stat += "motion_detection-- clip_file - " + buffFile + "\n";
for (auto i = 0; capPair(prev, next, capture, share); ++i) while (capPair(prev, next, capture, share))
{ {
share->stat += "frame_pair-- " + to_string(i) + "\n"; if (imgDiff(toGray(prev), toGray(next), share))
if (imgDiff(toGray(prev), toGray(next), block, share))
{ {
*img = next;
mod = true; break; mod = true; break;
} }
} }

View File

@ -14,6 +14,7 @@
// GNU General Public License for more details. // GNU General Public License for more details.
#include "common.h" #include "common.h"
#include "obj_detect.h"
struct sec_t struct sec_t
{ {
@ -26,7 +27,7 @@ struct sec_t
void secDiff(const Mat &imgA, const Mat &imgB, int rows, int cols, int rowOffs, int colOffs, vector<sec_t> *results, mutex *secMutex, shared_t *share); void secDiff(const Mat &imgA, const Mat &imgB, int rows, int cols, int rowOffs, int colOffs, vector<sec_t> *results, mutex *secMutex, shared_t *share);
bool pixDiff(const uchar &pixA, const uchar &pixB, shared_t *share); bool pixDiff(const uchar &pixA, const uchar &pixB, shared_t *share);
bool imgDiff(const Mat &prev, const Mat &next, Rect *block, shared_t *share); bool imgDiff(const Mat &prev, const Mat &next, shared_t *share);
bool moDetect(const string &buffFile, Rect *block, Mat *img, shared_t *share); bool moDetect(const string &buffFile, shared_t *share);
#endif // MO_DETECT_H #endif // MO_DETECT_H

View File

@ -12,53 +12,50 @@
#include "obj_detect.h" #include "obj_detect.h"
bool objectInImage(const Mat &src, const Rect &area, shared_t *share) bool objectFlowInImage(const Mat &inPrev, const Mat &inNext, const Rect &area, shared_t *share)
{ {
// reference: https://github.com/doleron/yolov5-opencv-cpp-python/blob/main/cpp/yolo.cpp Mat prev(inPrev, area);
Mat blockImage(src, area); Mat next(inNext, area);
Mat blob;
dnn::blobFromImage(blockImage, blob, 1./255., Size(area.width, area.height), cv::Scalar(), true, false); // optical flow calculations are used to detect motion.
// reference: https://docs.opencv.org/3.4/d4/dee/tutorial_optical_flow.html
vector<Point2f> p0, p1;
vector<uchar> status;
vector<float> err;
share->network.setInput(blob); auto criteria = TermCriteria((TermCriteria::COUNT) + (TermCriteria::EPS), 10, 0.03);
vector<Mat> outputs; // distance is basically 0.0578% of the total pixel area of the
// frames. this value is used later below.
auto thresh = ((double) 0.0578 / (double) 100) * (area.height * area.width);
auto count = 0;
share->network.forward(outputs, share->network.getUnconnectedOutLayersNames()); goodFeaturesToTrack(prev, p0, 100, 0.3, 7, Mat(), 7, false, 0.04);
calcOpticalFlowPyrLK(prev, next, p0, p1, status, err, Size(10, 10), 2, criteria);
float *data = (float *)outputs[0].data; share->stat += " object_flow_detection-- block-" + to_string(area.x) + "x" + to_string(area.y) + "\n";
share->stat += "object_detection---\n"; for(uint i = 0; i < p0.size(); i++)
share->stat += " outputs_size=" + to_string(outputs.size()) + " looking_for_objects...\n";
for (int i = 0; i < 25200; ++i)
{ {
// confidence level data[4]; // select good points
if (data[4] >= 0.4) if(status[i] == 1)
{ {
float *classesScores = data + 5; auto dis = norm(p0[i] - p1[i]);
Mat scores(1, share->classNames.size(), CV_32FC1, classesScores); if (dis > thresh)
Point classId;
double maxClassScore;
minMaxLoc(scores, 0, &maxClassScore, 0, &classId);
share->stat += " potential_object[confidence_level]=" + to_string(data[4]) + "\n";
share->stat += " potential_object[class_score]=" + to_string(maxClassScore) + "\n";
if (maxClassScore > 0.2)
{ {
share->stat += " object_confirmed\n"; // any points that moved 0.0578% or more of the total pixel
// area can be considered an object in motion.
share->stat += " obj_move_greater_than=" + to_string(thresh) + "\n";
share->stat += " obj_move_distance=" + to_string(dis) + "\n";
return true; return true;
} }
} }
data += 85;
} }
share->stat += " no_objects_found\n"; share->stat += " no_object_flow_detected\n";
return false; return false;
} }

View File

@ -15,6 +15,6 @@
#include "common.h" #include "common.h"
bool objectInImage(const Mat &image, const Rect &area, shared_t *share); bool objectFlowInImage(const Mat &inPrev, const Mat &inNext, const Rect &area, shared_t *share);
#endif // OBJ_DETECT_H #endif // OBJ_DETECT_H