Commit 7c7cf2a8 authored by vincent's avatar vincent

PredictAgeAndGenderTask

parent 4205fc29
......@@ -6,7 +6,7 @@ import { seperateWeightMaps } from '../faceProcessor/util';
import { TinyXception } from '../xception/TinyXception';
import { extractParams } from './extractParams';
import { extractParamsFromWeigthMap } from './extractParamsFromWeigthMap';
import { NetOutput, NetParams } from './types';
import { AgeAndGenderPrediction, Gender, NetOutput, NetParams } from './types';
export class AgeGenderNet extends NeuralNetwork<NetParams> {
......@@ -50,17 +50,32 @@ export class AgeGenderNet extends NeuralNetwork<NetParams> {
return this.forwardInput(await toNetInput(input))
}
public async predictAgeAndGender(input: TNetInput): Promise<{ age: number, gender: string, genderProbability: number }> {
public async predictAgeAndGender(input: TNetInput): Promise<AgeAndGenderPrediction | AgeAndGenderPrediction[]> {
const netInput = await toNetInput(input)
const out = await this.forwardInput(netInput)
const age = (await out.age.data())[0]
const probMale = (await out.gender.data())[0]
const isMale = probMale > 0.5
const gender = isMale ? 'male' : 'female'
const genderProbability = isMale ? probMale : (1 - probMale)
return { age, gender, genderProbability }
const ages = tf.unstack(out.age)
const genders = tf.unstack(out.gender)
const ageAndGenderTensors = ages.map((ageTensor, i) => ({
ageTensor,
genderTensor: genders[i]
}))
const predictionsByBatch = await Promise.all(
ageAndGenderTensors.map(async ({ ageTensor, genderTensor }) => {
const age = (await ageTensor.data())[0]
const probMale = (await out.gender.data())[0]
const isMale = probMale > 0.5
const gender = isMale ? Gender.MALE : Gender.FEMALE
const genderProbability = isMale ? probMale : (1 - probMale)
return { age, gender, genderProbability }
})
)
return netInput.isBatchInput
? predictionsByBatch
: predictionsByBatch[0]
}
protected getDefaultModelName(): string {
......
import * as tf from '@tensorflow/tfjs-core';
import { TfjsImageRecognitionBase } from 'tfjs-image-recognition-base';
export type AgeAndGenderPrediction = {
age: number
gender: Gender
genderProbability: number
}
export enum Gender {
FEMALE = 'female',
MALE = 'male'
}
export type NetOutput = { age: tf.Tensor1D, gender: tf.Tensor2D }
export type NetParams = {
......
export type WithAge<TSource> = TSource & {
age: number
}
export function isWithAge(obj: any): obj is WithAge<{}> {
return typeof obj['age'] === 'number'
}
export function extendWithAge<
TSource
> (
sourceObj: TSource,
age: number
): WithAge<TSource> {
const extension = { age }
return Object.assign({}, sourceObj, extension)
}
\ No newline at end of file
import { isValidProbablitiy } from 'tfjs-image-recognition-base';
import { Gender } from '../ageGenderNet/types';
export type WithGender<TSource> = TSource & {
gender: Gender
genderProbability: number
}
export function isWithGender(obj: any): obj is WithGender<{}> {
return (obj['gender'] === Gender.MALE || obj['gender'] === Gender.FEMALE)
&& isValidProbablitiy(obj['genderProbability'])
}
export function extendWithGender<
TSource
> (
sourceObj: TSource,
gender: Gender,
genderProbability: number
): WithGender<TSource> {
const extension = { gender, genderProbability }
return Object.assign({}, sourceObj, extension)
}
\ No newline at end of file
import * as tf from '@tensorflow/tfjs-core';
import { TNetInput } from 'tfjs-image-recognition-base';
import { extractFaces, extractFaceTensors } from '../dom';
import { extendWithFaceDescriptor, WithFaceDescriptor } from '../factories/WithFaceDescriptor';
import { WithFaceDetection } from '../factories/WithFaceDetection';
import { WithFaceLandmarks } from '../factories/WithFaceLandmarks';
import { ComposableTask } from './ComposableTask';
import { extractAllFacesAndComputeResults, extractSingleFaceAndComputeResult } from './extractFacesAndComputeResults';
import { nets } from './nets';
import {
PredictAllAgeAndGenderWithFaceAlignmentTask,
PredictSingleAgeAndGenderWithFaceAlignmentTask,
} from './PredictAgeAndGenderTask';
import {
PredictAllFaceExpressionsWithFaceAlignmentTask,
PredictSingleFaceExpressionsWithFaceAlignmentTask,
} from './PredictFaceExpressionsTask';
export class ComputeFaceDescriptorsTaskBase<TReturn, TParentReturn> extends ComposableTask<TReturn> {
constructor(
......@@ -25,19 +32,25 @@ export class ComputeAllFaceDescriptorsTask<
const parentResults = await this.parentTask
const dlibAlignedRects = parentResults.map(({ landmarks }) => landmarks.align(null, { useDlibAlignment: true }))
const dlibAlignedFaces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
? await extractFaceTensors(this.input, dlibAlignedRects)
: await extractFaces(this.input, dlibAlignedRects)
const descriptors = await extractAllFacesAndComputeResults<TSource, Float32Array[]>(
parentResults,
this.input,
faces => Promise.all(faces.map(face =>
nets.faceRecognitionNet.computeFaceDescriptor(face) as Promise<Float32Array>
)),
null,
parentResult => parentResult.landmarks.align(null, { useDlibAlignment: true })
)
const results = await Promise.all(parentResults.map(async (parentResult, i) => {
const descriptor = await nets.faceRecognitionNet.computeFaceDescriptor(dlibAlignedFaces[i]) as Float32Array
return extendWithFaceDescriptor<TSource>(parentResult, descriptor)
}))
return descriptors.map((descriptor, i) => extendWithFaceDescriptor<TSource>(parentResults[i], descriptor))
}
dlibAlignedFaces.forEach(f => f instanceof tf.Tensor && f.dispose())
withFaceExpressions(): PredictAllFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictAllFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
return results
withAgeAndGender(): PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
}
......@@ -51,15 +64,22 @@ export class ComputeSingleFaceDescriptorTask<
if (!parentResult) {
return
}
const descriptor = await extractSingleFaceAndComputeResult<TSource, Float32Array>(
parentResult,
this.input,
face => nets.faceRecognitionNet.computeFaceDescriptor(face) as Promise<Float32Array>,
null,
parentResult => parentResult.landmarks.align(null, { useDlibAlignment: true })
)
const dlibAlignedRect = parentResult.landmarks.align(null, { useDlibAlignment: true })
const alignedFaces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
? await extractFaceTensors(this.input, [dlibAlignedRect])
: await extractFaces(this.input, [dlibAlignedRect])
const descriptor = await nets.faceRecognitionNet.computeFaceDescriptor(alignedFaces[0]) as Float32Array
return extendWithFaceDescriptor(parentResult, descriptor)
}
alignedFaces.forEach(f => f instanceof tf.Tensor && f.dispose())
withFaceExpressions(): PredictSingleFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictSingleFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
return extendWithFaceDescriptor(parentResult, descriptor)
withAgeAndGender(): PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
}
\ No newline at end of file
......@@ -10,6 +10,10 @@ import { extendWithFaceLandmarks, WithFaceLandmarks } from '../factories/WithFac
import { ComposableTask } from './ComposableTask';
import { ComputeAllFaceDescriptorsTask, ComputeSingleFaceDescriptorTask } from './ComputeFaceDescriptorsTasks';
import { nets } from './nets';
import {
PredictAllAgeAndGenderWithFaceAlignmentTask,
PredictSingleAgeAndGenderWithFaceAlignmentTask,
} from './PredictAgeAndGenderTask';
import {
PredictAllFaceExpressionsWithFaceAlignmentTask,
PredictSingleFaceExpressionsWithFaceAlignmentTask,
......@@ -59,6 +63,10 @@ export class DetectAllFaceLandmarksTask<
return new PredictAllFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withAgeAndGender(): PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptors(): ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>> {
return new ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>>(this, this.input)
}
......@@ -91,6 +99,10 @@ export class DetectSingleFaceLandmarksTask<
return new PredictSingleFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withAgeAndGender(): PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptor(): ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>> {
return new ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>>(this, this.input)
}
......
import * as tf from '@tensorflow/tfjs-core';
import { TNetInput } from 'tfjs-image-recognition-base';
import { AgeAndGenderPrediction } from '../ageGenderNet/types';
import { extendWithAge, WithAge } from '../factories/WithAge';
import { WithFaceDetection } from '../factories/WithFaceDetection';
import { WithFaceLandmarks } from '../factories/WithFaceLandmarks';
import { extendWithGender, WithGender } from '../factories/WithGender';
import { ComposableTask } from './ComposableTask';
import { ComputeAllFaceDescriptorsTask, ComputeSingleFaceDescriptorTask } from './ComputeFaceDescriptorsTasks';
import { extractAllFacesAndComputeResults, extractSingleFaceAndComputeResult } from './extractFacesAndComputeResults';
import { nets } from './nets';
import {
PredictAllFaceExpressionsWithFaceAlignmentTask,
PredictSingleFaceExpressionsWithFaceAlignmentTask,
} from './PredictFaceExpressionsTask';
export class PredictAgeAndGenderTaskBase<TReturn, TParentReturn> extends ComposableTask<TReturn> {
constructor(
protected parentTask: ComposableTask<TParentReturn> | Promise<TParentReturn>,
protected input: TNetInput,
protected extractedFaces?: Array<HTMLCanvasElement | tf.Tensor3D>
) {
super()
}
}
export class PredictAllAgeAndGenderTask<
TSource extends WithFaceDetection<{}>
> extends PredictAgeAndGenderTaskBase<WithAge<WithGender<TSource>>[], TSource[]> {
public async run(): Promise<WithAge<WithGender<TSource>>[]> {
const parentResults = await this.parentTask
const ageAndGenderByFace = await extractAllFacesAndComputeResults<TSource, AgeAndGenderPrediction[]>(
parentResults,
this.input,
async faces => await Promise.all(faces.map(
face => nets.ageGenderNet.predictAgeAndGender(face) as Promise<AgeAndGenderPrediction>
)),
this.extractedFaces
)
return parentResults.map((parentResult, i) => {
const { age, gender, genderProbability } = ageAndGenderByFace[i]
return extendWithAge(extendWithGender(parentResult, gender, genderProbability), age)
})
}
}
export class PredictSingleAgeAndGenderTask<
TSource extends WithFaceDetection<{}>
> extends PredictAgeAndGenderTaskBase<WithAge<WithGender<TSource>> | undefined, TSource | undefined> {
public async run(): Promise<WithAge<WithGender<TSource>> | undefined> {
const parentResult = await this.parentTask
if (!parentResult) {
return
}
const { age, gender, genderProbability } = await extractSingleFaceAndComputeResult<TSource, AgeAndGenderPrediction>(
parentResult,
this.input,
face => nets.ageGenderNet.predictAgeAndGender(face) as Promise<AgeAndGenderPrediction>,
this.extractedFaces
)
return extendWithAge(extendWithGender(parentResult, gender, genderProbability), age)
}
}
export class PredictAllAgeAndGenderWithFaceAlignmentTask<
TSource extends WithFaceLandmarks<WithFaceDetection<{}>>
> extends PredictAllAgeAndGenderTask<TSource> {
withFaceExpressions(): PredictAllFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictAllFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptors(): ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>> {
return new ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>>(this, this.input)
}
}
export class PredictSingleAgeAndGenderWithFaceAlignmentTask<
TSource extends WithFaceLandmarks<WithFaceDetection<{}>>
> extends PredictSingleAgeAndGenderTask<TSource> {
withFaceExpressions(): PredictSingleFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictSingleFaceExpressionsWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptor(): ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>> {
return new ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>>(this, this.input)
}
}
\ No newline at end of file
import * as tf from '@tensorflow/tfjs-core';
import { TNetInput } from 'tfjs-image-recognition-base';
import { extractFaces, extractFaceTensors } from '../dom';
import { FaceExpressions } from '../faceExpressionNet/FaceExpressions';
import { WithFaceDetection } from '../factories/WithFaceDetection';
import { extendWithFaceExpressions, WithFaceExpressions } from '../factories/WithFaceExpressions';
import { isWithFaceLandmarks, WithFaceLandmarks } from '../factories/WithFaceLandmarks';
import { WithFaceLandmarks } from '../factories/WithFaceLandmarks';
import { ComposableTask } from './ComposableTask';
import { ComputeAllFaceDescriptorsTask, ComputeSingleFaceDescriptorTask } from './ComputeFaceDescriptorsTasks';
import { extractAllFacesAndComputeResults, extractSingleFaceAndComputeResult } from './extractFacesAndComputeResults';
import { nets } from './nets';
import {
PredictAllAgeAndGenderWithFaceAlignmentTask,
PredictSingleAgeAndGenderWithFaceAlignmentTask,
} from './PredictAgeAndGenderTask';
export class PredictFaceExpressionsTaskBase<TReturn, TParentReturn> extends ComposableTask<TReturn> {
constructor(
protected parentTask: ComposableTask<TParentReturn> | Promise<TParentReturn>,
protected input: TNetInput
protected input: TNetInput,
protected extractedFaces?: Array<HTMLCanvasElement | tf.Tensor3D>
) {
super()
}
......@@ -27,18 +32,14 @@ export class PredictAllFaceExpressionsTask<
const parentResults = await this.parentTask
const faceBoxes = parentResults.map(
parentResult => isWithFaceLandmarks(parentResult) ? parentResult.alignedRect : parentResult.detection
const faceExpressionsByFace = await extractAllFacesAndComputeResults<TSource, FaceExpressions[]>(
parentResults,
this.input,
async faces => await Promise.all(faces.map(
face => nets.faceExpressionNet.predictExpressions(face) as Promise<FaceExpressions>
)),
this.extractedFaces
)
const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
? await extractFaceTensors(this.input, faceBoxes)
: await extractFaces(this.input, faceBoxes)
const faceExpressionsByFace = await Promise.all(faces.map(
face => nets.faceExpressionNet.predictExpressions(face)
)) as FaceExpressions[]
faces.forEach(f => f instanceof tf.Tensor && f.dispose())
return parentResults.map(
(parentResult, i) => extendWithFaceExpressions<TSource>(parentResult, faceExpressionsByFace[i])
......@@ -57,14 +58,12 @@ export class PredictSingleFaceExpressionsTask<
return
}
const faceBox = isWithFaceLandmarks(parentResult) ? parentResult.alignedRect : parentResult.detection
const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
? await extractFaceTensors(this.input, [faceBox])
: await extractFaces(this.input, [faceBox])
const faceExpressions = await nets.faceExpressionNet.predictExpressions(faces[0]) as FaceExpressions
faces.forEach(f => f instanceof tf.Tensor && f.dispose())
const faceExpressions = await extractSingleFaceAndComputeResult<TSource, FaceExpressions>(
parentResult,
this.input,
face => nets.faceExpressionNet.predictExpressions(face) as Promise<FaceExpressions>,
this.extractedFaces
)
return extendWithFaceExpressions(parentResult, faceExpressions)
}
......@@ -74,6 +73,10 @@ export class PredictAllFaceExpressionsWithFaceAlignmentTask<
TSource extends WithFaceLandmarks<WithFaceDetection<{}>>
> extends PredictAllFaceExpressionsTask<TSource> {
withAgeAndGender(): PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictAllAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptors(): ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>> {
return new ComputeAllFaceDescriptorsTask<WithFaceLandmarks<TSource>>(this, this.input)
}
......@@ -83,6 +86,10 @@ export class PredictSingleFaceExpressionsWithFaceAlignmentTask<
TSource extends WithFaceLandmarks<WithFaceDetection<{}>>
> extends PredictSingleFaceExpressionsTask<TSource> {
withAgeAndGender(): PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>> {
return new PredictSingleAgeAndGenderWithFaceAlignmentTask<WithFaceLandmarks<TSource>>(this, this.input)
}
withFaceDescriptor(): ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>> {
return new ComputeSingleFaceDescriptorTask<WithFaceLandmarks<TSource>>(this, this.input)
}
......
import * as tf from '@tensorflow/tfjs-core';
import { TNetInput } from 'tfjs-image-recognition-base';
import { FaceDetection } from '../classes/FaceDetection';
import { extractFaces, extractFaceTensors } from '../dom';
import { WithFaceDetection } from '../factories/WithFaceDetection';
import { isWithFaceLandmarks, WithFaceLandmarks } from '../factories/WithFaceLandmarks';
export async function extractAllFacesAndComputeResults<TSource extends WithFaceDetection<{}>, TResult>(
parentResults: TSource[],
input: TNetInput,
computeResults: (faces: Array<HTMLCanvasElement | tf.Tensor3D>) => Promise<TResult>,
extractedFaces?: Array<HTMLCanvasElement | tf.Tensor3D> | null,
getRectForAlignment: (parentResult: WithFaceLandmarks<TSource, any>) => FaceDetection = ({ alignedRect }) => alignedRect
) {
const faceBoxes = parentResults.map(parentResult =>
isWithFaceLandmarks(parentResult)
? getRectForAlignment(parentResult)
: parentResult.detection
)
const faces: Array<HTMLCanvasElement | tf.Tensor3D> = extractedFaces || (
input instanceof tf.Tensor
? await extractFaceTensors(input, faceBoxes)
: await extractFaces(input, faceBoxes)
)
const results = await computeResults(faces)
faces.forEach(f => f instanceof tf.Tensor && f.dispose())
return results
}
export async function extractSingleFaceAndComputeResult<TSource extends WithFaceDetection<{}>, TResult>(
parentResult: TSource,
input: TNetInput,
computeResult: (face: HTMLCanvasElement | tf.Tensor3D) => Promise<TResult>,
extractedFaces?: Array<HTMLCanvasElement | tf.Tensor3D> | null,
getRectForAlignment?: (parentResult: WithFaceLandmarks<TSource, any>) => FaceDetection
) {
return extractAllFacesAndComputeResults<TSource, TResult>(
[parentResult],
input,
async faces => computeResult(faces[0]),
extractedFaces,
getRectForAlignment
)
}
\ No newline at end of file
import { TfjsImageRecognitionBase, TNetInput } from 'tfjs-image-recognition-base';
import { AgeGenderNet } from '../ageGenderNet/AgeGenderNet';
import { AgeAndGenderPrediction } from '../ageGenderNet/types';
import { FaceDetection } from '../classes/FaceDetection';
import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
......@@ -26,7 +28,8 @@ export const nets = {
faceLandmark68Net: new FaceLandmark68Net(),
faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
faceRecognitionNet: new FaceRecognitionNet(),
faceExpressionNet: new FaceExpressionNet()
faceExpressionNet: new FaceExpressionNet(),
ageGenderNet: new AgeGenderNet()
}
/**
......@@ -107,16 +110,25 @@ export const computeFaceDescriptor = (input: TNetInput): Promise<Float32Array |
/**
* Recognizes the facial expressions of a face and returns the likelyhood of
* each facial expression.
* Recognizes the facial expressions from a face image.
*
* @param inputs The face image extracted from the bounding box of a face. Can
* also be an array of input images, which will be batch processed.
* @returns An array of facial expressions with corresponding probabilities or array thereof in case of batch input.
* @returns Facial expressions with corresponding probabilities or array thereof in case of batch input.
*/
export const recognizeFaceExpressions = (input: TNetInput): Promise<FaceExpressions | FaceExpressions[]> =>
nets.faceExpressionNet.predictExpressions(input)
/**
* Predicts age and gender from a face image.
*
* @param inputs The face image extracted from the bounding box of a face. Can
* also be an array of input images, which will be batch processed.
* @returns Predictions with age, gender and gender probability or array thereof in case of batch input.
*/
export const predictAgeAndGender = (input: TNetInput): Promise<AgeAndGenderPrediction | AgeAndGenderPrediction[]> =>
nets.ageGenderNet.predictAgeAndGender(input)
export const loadSsdMobilenetv1Model = (url: string) => nets.ssdMobilenetv1.load(url)
export const loadTinyFaceDetectorModel = (url: string) => nets.tinyFaceDetector.load(url)
export const loadMtcnnModel = (url: string) => nets.mtcnn.load(url)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment