{"feature_extractor_name":"audio_fingerprint_extractor","version":"v1","feature_extractor_id":"audio_fingerprint_extractor_v1","description":"Audio fingerprinting using CLAP (Contrastive Language-Audio Pretraining).\n\nExtracts 512-dimensional embeddings from audio files or audio tracks extracted from video. Uses laion/clap-htsat-tiny model.\n\n**Pipeline Steps:**\n1. Audio extraction from video (if video input, via FFmpeg)\n2. Audio segmentation (configurable window + hop)\n3. CLAP embedding generation (512-d per segment)\n4. L2 normalization\n5. Output validation\n\n**Use for:** Sound mark detection, audio similarity search, music/jingle identification, audio deduplication.\n\n**Not for:** Speech-to-text (use omnilingual_asr), general audio classification (use dedicated classifiers).","icon":"music","category":"audio","source":"builtin","type_mode":"multimodal","expected_input_types":null,"inference_type":"embedding","input_schema":{"description":"Input schema for audio fingerprint extractor.\n\nProvide an audio file or a video file (audio track is extracted).","examples":[{"audio":"s3://bucket/audio/sound-mark.mp3","description":"Audio file"},{"description":"Video with audio track","video":"s3://bucket/videos/commercial.mp4"}],"properties":{"audio":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Audio URL or S3 path. Formats: MP3, WAV, FLAC, OGG, M4A. ","examples":["s3://bucket/audio/jingle.mp3"],"title":"Audio"},"video":{"anyOf":[{"type":"string"},{"type":"null"}],"default":null,"description":"Video URL or S3 path. Audio track is extracted via FFmpeg. Formats: MP4, MOV, AVI, MKV, WebM.","examples":["s3://bucket/videos/ad-spot.mp4"],"title":"Video"}},"title":"AudioFingerprintExtractorInput","type":"object"},"output_schema":{"additionalProperties":true,"description":"Output schema for audio fingerprint extractor.\n\nEach document represents ONE audio segment with its CLAP embedding.\nA single input may produce multiple segments (sliding window).","properties":{"audio_fingerprint_extractor_v1_embedding":{"description":"CLAP audio embedding (512-d, L2-normalized). Use cosine similarity for audio matching.","items":{"type":"number"},"maxItems":512,"minItems":512,"title":"Audio Fingerprint Extractor V1 Embedding","type":"array"},"segment_index":{"description":"Index of this segment in the source audio (0-based)","title":"Segment Index","type":"integer"},"start_time_sec":{"description":"Start time of this segment in the source audio (seconds)","title":"Start Time Sec","type":"number"},"end_time_sec":{"description":"End time of this segment in the source audio (seconds)","title":"End Time Sec","type":"number"},"duration_sec":{"description":"Duration of this segment (seconds)","title":"Duration Sec","type":"number"},"total_duration_sec":{"anyOf":[{"type":"number"},{"type":"null"}],"default":null,"description":"Total duration of the source audio (seconds)","title":"Total Duration Sec"},"sample_rate":{"description":"Sample rate used for processing","title":"Sample Rate","type":"integer"},"audio_source_type":{"description":"Source type: 'audio' or 'video'","title":"Audio Source Type","type":"string"},"embedding_model":{"default":"laion/clap-htsat-tiny","description":"CLAP model used for embedding","title":"Embedding Model","type":"string"},"processing_time_ms":{"description":"Processing time for this segment (milliseconds)","title":"Processing Time Ms","type":"number"}},"required":["audio_fingerprint_extractor_v1_embedding","segment_index","start_time_sec","end_time_sec","duration_sec","sample_rate","audio_source_type","processing_time_ms"],"title":"AudioFingerprintExtractorOutput","type":"object"},"parameter_schema":{"description":"Parameters for the Audio Fingerprint Extractor.\n\nProcesses audio files (or audio extracted from video) through CLAP\n(Contrastive Language-Audio Pretraining) to produce 512-d embeddings\nsuitable for audio fingerprint matching.\n\nCore Pipeline:\n1. Audio extraction (if video input, via FFmpeg)\n2. Segmentation into fixed-length windows\n3. CLAP embedding (laion/clap-htsat-tiny, 512-d)\n4. L2 normalization\n\nUse Cases:\n    - Sound mark detection (IP safety)\n    - Audio similarity search\n    - Music/jingle identification\n    - Audio deduplication","examples":[{"description":"Sound mark detection (IP safety)","extractor_type":"audio_fingerprint_extractor","normalize_embeddings":true,"sample_rate":48000,"segment_duration_sec":5.0,"segment_hop_sec":2.5},{"description":"Music identification (longer segments)","extractor_type":"audio_fingerprint_extractor","sample_rate":48000,"segment_duration_sec":10.0,"segment_hop_sec":5.0}],"properties":{"extractor_type":{"const":"audio_fingerprint_extractor","default":"audio_fingerprint_extractor","description":"Discriminator field. Must be 'audio_fingerprint_extractor'.","title":"Extractor Type","type":"string"},"segment_duration_sec":{"default":5.0,"description":"Duration of each audio segment in seconds. 5.0: Recommended for sound mark matching. Shorter segments increase recall but reduce per-segment context.","maximum":30.0,"minimum":1.0,"title":"Segment Duration Sec","type":"number"},"segment_hop_sec":{"default":2.5,"description":"Hop size between segments in seconds. 2.5: 50% overlap (recommended). Set equal to segment_duration_sec for no overlap.","maximum":15.0,"minimum":0.5,"title":"Segment Hop Sec","type":"number"},"sample_rate":{"default":48000,"description":"Target sample rate for audio. 48000: CLAP default (recommended). Audio is resampled to this rate before embedding.","title":"Sample Rate","type":"integer"},"normalize_embeddings":{"default":true,"description":"L2-normalize embeddings to unit vectors (recommended for cosine similarity).","title":"Normalize Embeddings","type":"boolean"},"max_audio_length_sec":{"default":120.0,"description":"Maximum audio length to process in seconds. 120: Default (2 minutes). Audio beyond this is truncated.","maximum":600.0,"minimum":1.0,"title":"Max Audio Length Sec","type":"number"}},"title":"AudioFingerprintExtractorParams","type":"object"},"supported_input_types":["audio","video"],"max_inputs":{"audio":1,"video":1},"default_parameters":{},"costs":{"tier":2,"tier_label":"STANDARD","rates":[{"unit":"extraction","credits_per_unit":3,"description":"Cost per audio segment processed through CLAP embedding"}]},"required_vector_indexes":[{"feature_uri":"mixpeek://audio_fingerprint_extractor@v1/laion__clap_htsat_tiny","name":"laion__clap_htsat_tiny","description":"CLAP HTSAT-tiny audio embedding for audio fingerprint matching","type":"single","index":{"name":"audio_fingerprint_extractor_v1_embedding","description":"512-d CLAP audio embedding for audio fingerprint matching. Supports cosine similarity search against audio corpus.","dimensions":512,"type":"dense","distance":"Cosine","datatype":"float32","on_disk":null,"supported_inputs":["audio"],"inference_name":"laion__clap_htsat_tiny","inference_service_id":"laion/clap-htsat-tiny","purpose":null,"vector_name_override":null,"supports_multi_query":false}}],"required_payload_indexes":[],"position_fields":["segment_index"],"capabilities":["batch","realtime"],"example_usage":{"namespace":{"feature_extractors":[{"name":"audio_fingerprint_extractor","version":"v1"}]},"collection":{"feature_extractor":{"name":"audio_fingerprint_extractor","version":"v1","input_mappings":{"audio":"<your_audio_field>","video":"<your_video_field>"},"parameters":{"segment_duration_sec":5.0,"segment_hop_sec":2.5,"sample_rate":48000,"normalize_embeddings":true,"max_audio_length_sec":120.0}}}}}