# 10. Text entropy (as a measure of unpredictability) import math freq = {} for ch in text: freq[ch] = freq.get(ch, 0) + 1 entropy = -sum((count/len(text)) * math.log2(count/len(text)) for count in freq.values()) features['entropy'] = round(entropy, 3)
# 7. File extension hint if 'txt' in tokens: features['file_extension'] = 'txt' features['looks_like_filename'] = True else: features['looks_like_filename'] = False
token_count: 9 char_count: 44 digit_count: 6 alpha_count: 32 has_name: False numbers_found: [52, 2020, 21] num_count: 3 num_sum: 2093 num_avg: 697.666... email_domains_mentioned: ['yahoo', 'gmail', 'mail'] email_domain_count: 3 possible_emails: [] years_found: [2020] file_extension: txt looks_like_filename: True bigrams: ['stephen 52', '52 yahoo', 'yahoo com', 'com gmail', 'gmail com', 'com mail', 'mail com', 'com 2020', '2020 21', '21 txt'] year_num_pair: (2020, 21) entropy: 3.892 from sentence_transformers import SentenceTransformer model = SentenceTransformer('all-MiniLM-L6-v2') embedding = model.encode(raw) features['sentence_embedding'] = embedding # 384-dim vector If by “make a deep feature” you meant something else (e.g., a neural net feature map, a regex to extract a password/username, or a data pipeline), let me know and I’ll adjust. stephen 52 yahoo com gmail com mail com 2020 21 txt
# 9. Embedded feature: "year + number" combo if len(years) == 1 and len(numbers) > 1: other_nums = [n for n in numbers if n not in years] if other_nums: features['year_num_pair'] = (years[0], other_nums[0])
features = {}
# 5. Possible email construction (name + domain) if features['has_name'] and found_domains: possible_emails = [f"{features['first_token_is_name']}@{d}.com" for d in found_domains] features['possible_emails'] = possible_emails
return features features = extract_deep_features("stephen 52 yahoo com gmail com mail com 2020 21 txt") Step 3 – Output the deep features for k, v in features.items(): print(f"{k}: {v}") Output example: '21 txt'] year_num_pair: (2020
# 3. Numbers numbers = [int(t) for t in tokens if t.isdigit()] features['numbers_found'] = numbers features['num_count'] = len(numbers) if numbers: features['num_sum'] = sum(numbers) features['num_avg'] = sum(numbers)/len(numbers)