# -*- coding: utf-8 -*- """ 拼多多酱腌菜行业运营数据看板 - 数据构建脚本（v7 版） 相对 v6 的改进： A. 热门主题升级为五大池（当季新品/热门跟款/全网热销/潜力商品/热门收藏，THEME_FILES=5 项） B. 消费者正在搜·站内热搜/站内缺货词表（search_analysis.json）并入数据结构： consumer_search.hot_related / short_related / excluded / hot_count / short_count / collected_at C. 关键词分词拆解：长标题拆二级词，扩充蓝海词库（不再只依赖菜单词） D. 关键词趋势环比：与上一日 data/hot_words.json 对比 score/freq/竞争度，标记「新增蓝海/新晋红海/热度上涨」 E. 表格曝光/点击分组：高曝光低点击=主图/标题短板机会；低曝光高点击=值得加推 F. 蓝海词突变预警：当日蓝海词数量与上一日对比，减少超 30% 写入预警 G. 榜单时效标注：rank_lists 带 captured_at 实时时间 + source 说明 H. 全部时间戳使用实时时间（collected_at/generated_at/captured_at） 竞品关键词搜索池（kw_*.json）已按用户指令停用（SKIP_KW_POOLS=True）。 """ import json, collections, os, datetime, re, math BASE = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(BASE, "data") SCRAPE_DIR = os.path.join(BASE, "_scrape") os.makedirs(DATA_DIR, exist_ok=True) now = datetime.datetime.now() now_str = now.strftime("%Y-%m-%d %H:%M") today_str = now.strftime("%Y-%m-%d") LOG = [] def log(msg): LOG.append(msg) print(msg) # ============ 1. 加载热门主题四大池并合并去重（2026-10-11 用户指令：热门跟款池平台仅放约35条、噪声大已移除，只采当季新品/全网热销/潜力商品/热门收藏） ============ THEME_FILES = [ ("当季新品", "season_new.json"), ("全网热销", "hot_sale.json"), ("潜力商品", "potential.json"), ("热门收藏", "favorite.json"), ] MENU_COUNTS = { "萝卜": 2000, "排骨": 2000, "牛肉酱": 1935, "香菇": 1798, "石磨": 1529, "笋丝": 1503, "梅菜扣肉": 1364, "猪大肠": 1054, "莴笋": 899, "白萝卜": 746, "青萝卜": 649, "韭菜花": 626, "沙姜": 583, "辣椒水": 252, } # 竞品关键词搜索池规模（当日共搜到N，来自热点运营菜单模拟搜索） COMPETITOR_KWS = ["豆豉", "牛肉酱", "榨菜", "辣椒酱", "雪菜", "外婆菜", "红油豆角"] # 用户要求（2026-10-08）：删除看板「竞品关键词搜索池」菜单10，定时任务不再执行搜索关键词； # 置 True 后 build 跳过词池读取与 kw_pools 输出（数据文件保持干净），load_kw_pool 函数保留备用。 SKIP_KW_POOLS = True goods = {} raw_records = 0 season_new = 0 theme_collected = {} for theme, fname in THEME_FILES: fp = os.path.join(SCRAPE_DIR, fname) if not os.path.exists(fp): log("MISSING: " + fname) continue with open(fp, "r", encoding="utf-8") as f: bucket = json.load(f) theme_collected[theme] = len(bucket) for key, g in bucket.items(): raw_records += 1 name = g.get("name", "") price = float(g.get("price", 0) or 0) tags = g.get("tags", []) if not name: continue gkey = name + "|" + str(price) if gkey not in goods: goods[gkey] = {"name": name, "price": price, "tags": tags, "hot": g.get("hot", ""), "themes": []} if theme not in goods[gkey]["themes"]: goods[gkey]["themes"].append(theme) goods_list = list(goods.values()) season_new = sum(1 for g in goods_list if "当季新品" in g.get("tags", []) or "当季新品" in g.get("themes", [])) for g in goods_list: try: g["hot_val"] = int(str(g.get("hot", "")).replace("热度", "").strip() or 0) except Exception: g["hot_val"] = 0 g.pop("hot", None) # ============ 2. 类目与价格带 ============ def _guess_cat(name): cat_rules = [ ("榨菜", ["榨菜", "涪陵"]), ("咸菜/下饭菜", ["咸菜", "下饭", "小菜"]), ("酸菜", ["酸菜"]), ("萝卜干", ["萝卜"]), ("贡菜", ["贡菜"]), ("梅菜/外婆菜", ["梅菜", "外婆菜", "扣肉"]), ("泡椒/泡菜", ["泡椒", "泡菜", "凤爪", "泡萝卜"]), ("辣椒酱", ["辣椒", "辣酱", "剁椒", "红油", "椒"]), ("豆豉", ["豆豉", "豆鼓", "酱豆", "西瓜酱"]), ("牛肉/香菇酱", ["牛肉", "香菇", "肉酱"]), ("芝麻/花生酱", ["五仁", "花生", "芝麻", "麻酱"]), ("复合蘸料", ["蘸", "火锅底料", "香锅酱", "烧烤酱", "麻婆", "调料", "调味", "火锅蘸"]), ("其他", []), ] for cat, keys in cat_rules: if any(k in name for k in keys): return cat return "其他" for g in goods_list: g["cat"] = _guess_cat(g["name"]) GOOD_CATS = {"咸菜/下饭菜", "萝卜干", "辣椒酱", "复合蘸料", "牛肉/香菇酱", "榨菜", "梅菜/外婆菜", "泡椒/泡菜", "豆豉", "芝麻/花生酱", "酸菜", "贡菜"} # —— 主营相关过滤（2026-10-09 用户指令：仅保留酱腌菜/调味酱料/豆豉豆制品相关数据与关键词）—— # 硬排除：腌腊肉制品/蛋类/海鲜水产/休闲零食/饮料冲调/罐头速食等与主营无关品类，无条件剔除 HARD_EXCLUDE = (r"凤爪|鸡爪|鸭脖|鸭翅|鸭掌|鸭舌|鸡翅|鸡腿|鸡胗|卤蛋|鹌鹑蛋|无骨鸡爪|泡椒凤爪" r"|腊肉|腊肠|腊鱼|香肠|火腿肠|午餐肉|肉松|牛肉干|肉干|鱼干|咸鸭蛋|皮蛋|卤鸭" r"|海带|紫菜|海苔|鱿鱼|虾皮|虾米|虾仁|扇贝|花甲|海参|鲍鱼|鱼豆腐|豆腐串" r"|坚果|瓜子|核桃|开心果|果干|蜜饯|葡萄干|饼干|面包|蛋糕|月饼|粽子|辣条|辣片|豆腐串" r"|菜籽油|大豆油|花生油|玉米油|葵花籽油|食用油|调和油|压榨油|一级油|非转基因油|5L|5l" r"|饮料|果汁|奶茶|咖啡|茶叶|茶包|燕麦|麦片|藕粉|龟苓膏|八宝粥|自热火锅|自热米饭|速食粥|冲调|罐头" # 2026-10-09 用户新模板：站内热搜/站内缺货两池各采集1000+，过滤主营无关（炸锅烤箱类速食半成品/非食品杂物全剔除） r"|炸锅|烤箱|空气炸|薯条|酥肉|炸鸡|鸡柳|鸡排|鸡米|披萨|蛋挞|锡纸|烤盘|烘焙|预拌|蒸锅|电烤|炸串|天妇罗|裹粉|油炸|炸物|脆皮|油炸机" r"|手镯|玉镯|翡翠|玉石|床架|高达|手办|模型|情趣|自慰|玩具|窗帘|衣柜|礼服|卫衣|拖鞋|饰品|首饰|双肩包|帐篷|灯具|收纳|货架|衣架|签子|油纸|纸杯|纸盘|餐盒|微波|风炉|养生壶|电饭煲|取暖|加湿|扫地|纸巾|湿巾|床单|被套|面膜|口红|香水|洗发|沐浴|纸尿裤|奶粉|宠物|猫粮|狗粮|渔具|钓具|健身|哑铃|瑜伽|行李箱|钱包|钥匙扣|挂饰|摆件|香薰|花盆|绿植|多肉|花瓶|对联|贴纸|水槽|水龙头|花洒|瓷砖|地板|墙纸|螺丝|扳手|锤子|文具|铅笔|笔记本|词典|印章|茶壶|茶杯|茶具|咖啡机|榨汁机|破壁机|高压锅|炒锅|煎锅|汤锅|奶锅|不粘锅|铸铁锅|床罩|床头罩|盖布|礼盒|包装盒|保鲜袋|密封袋|压缩袋|收纳袋|垃圾袋") # 软排除：面食速食/干货香料，命中且无主营信号词时剔除（含酱类/咸菜等信号词则保留，如"牛肉香菇酱""剁椒鱼头调料"） SOFT_EXCLUDE = (r"方便面|泡面|意面|意大利面|速食面|挂面|螺蛳粉|酸辣粉|米粉|米线|粉丝|面皮|凉皮|拉面" r"|香料|花椒|八角|桂皮|香叶|卤料|干货|关东煮|冒菜|麻辣烫") MAIN_SIGNAL = r"酱|咸菜|榨菜|下饭|蘸|调料|调味|豆豉|豆鼓|酸菜|萝卜干|外婆菜|泡菜|泡椒|剁椒|辣椒|红油|豆角|洋姜|贡菜|橄榄菜|梅菜|牛肉|香菇|芝麻酱|花生酱|麻酱|豉" def _not_excluded(name): if re.search(HARD_EXCLUDE, name): return False # 面食速食硬过滤：标题含泡面/拉面/板面/火鸡面/即食等速食场景词，且无具体酱菜/酱料/豆豉主营品名时剔除 if re.search(r"泡面|拉面|板面|火鸡面|蟹黄面|螺蛳|酸辣粉|米线|面皮|凉皮|速食面|关东煮|麻辣烫|自热|即食|方便", name) \ and not re.search(r"下饭酱|拌面酱|牛肉酱|香菇酱|辣椒酱|豆瓣酱|芝麻酱|花生酱|豆豉|豆鼓|蒜蓉酱|辣酱|调味酱|下饭菜|咸菜|泡菜|酸菜|萝卜干|外婆菜|榨菜|洋姜|豆角|贡菜|橄榄菜|梅菜", name): return False if re.search(SOFT_EXCLUDE, name) and not re.search(MAIN_SIGNAL, name): return False return True # 热搜词黑名单：与主营无关的关键词（硬菜/海鲜/速食/零食等），不进入热搜与推荐 KEYWORD_BLACK = (r"排骨|猪肚|猪大肠|肥肠|红烧肉|牛腩|羊肉串|羊杂|肉丝|佛跳墙|龙虾|花甲|花胶" r"|鲍鱼|海参|章鱼|黑鱼|寿司|火鸡|方便面|泡面|螺蛳|酸辣粉|米粉|米线|挂面|自热|八宝粥" r"|腊肉|香肠|火腿|海鲜|海带|紫菜|鱿鱼|虾|鱼干|肉干|坚果|瓜子|饼干|面包|蛋糕|月饼|粽子" r"|罐头|饮料|果汁|咖啡|茶叶|燕麦|麦片|辣条|木耳|银耳|奶茶|藕粉|凉皮|凤爪|鸡爪|鸭脖|皮蛋|咸鸭蛋" r"|亚麻|金砖|山茶油|福神|玉溪|琥珀") raw_merged = len(goods_list) goods_list = [g for g in goods_list if g["cat"] in GOOD_CATS] goods_list = [g for g in goods_list if _not_excluded(g["name"])] log("pool_filter: merged=" + str(raw_merged) + " -> relevant=" + str(len(goods_list))) cat_stat = collections.Counter(g["cat"] for g in goods_list) def price_band(p): if p < 5: return "0-5元" if p < 10: return "5-10元" if p < 20: return "10-20元" if p < 50: return "20-50元" if p < 100: return "50-100元" return "100元以上" price_bands = collections.Counter(price_band(g["price"]) for g in goods_list) # ============ 2.1 P0：包装规格 / 净含量 / 商用家用 ============ PACK_RULES = [ ("整箱/整件", ["整箱", "整件", "整包", "50包", "箱装"]), ("罐装", ["罐装", "罐"]), ("瓶装", ["瓶装", "瓶"]), ("桶装", ["桶装", "桶"]), ("碗装", ["碗装", "碗"]), ("袋装", ["袋装", "袋", "包"]), ] COMMERCIAL_KEYS = ["商用", "批发", "餐饮", "食堂", "酒店", "大包装", "整件", "整箱", "摆摊", "面馆", "汤粉", "餐厅", "外卖"] FAMILY_KEYS = ["家用", "家庭", "便携", "小袋", "学生", "独立装", "随身"] def parse_pack(title): for pack, keys in PACK_RULES: if any(k in title for k in keys): return pack return "其他" def parse_weight_band(title): m = re.search(r"(\d+(?:\.\d+)?)\s*(g|克|kg|千克|斤|ml|毫升|L|升)", title) if not m: return "未标注" v = float(m.group(1)) unit = m.group(2) if unit in ("g", "克"): grams = v elif unit in ("ml", "毫升"): grams = v * 1.03 elif unit in ("kg", "千克"): grams = v * 1000 elif unit == "L": grams = v * 1000 else: # 斤 grams = v * 500 if grams < 100: return "100g以下" if grams < 300: return "100-300g" if grams < 500: return "300-500g" if grams < 1000: return "500g-1kg" if grams < 3000: return "1-3kg" return "3kg以上" def parse_usage(title): com = any(k in title for k in COMMERCIAL_KEYS) fam = any(k in title for k in FAMILY_KEYS) if com and fam: return "商用+家用" if com: return "商用" if fam: return "家用" return "通用" pack_stat = collections.Counter() weight_stat = collections.Counter() usage_stat = collections.Counter() for g in goods_list: t = g["name"] g["pack"] = parse_pack(t) g["weight_band"] = parse_weight_band(t) g["usage"] = parse_usage(t) ATTR_KEYS = ["农家", "无添加", "低盐", "老坛", "脆", "爽", "香辣", "麻辣", "酸辣", "开胃", "即食", "下饭", "泡椒", "剁椒", "豆豉", "手工", "秘制", "传统", "特产", "新鲜", "整箱", "批发", "商用", "拌面", "拌饭", "夹馍"] attr_stat = collections.OrderedDict() for k in ATTR_KEYS: c = sum(1 for g in goods_list if k in g["name"]) if c > 0: attr_stat[k] = c attr_top = sorted(attr_stat.items(), key=lambda x: -x[1])[:15] # ============ 2.2 P1：表格销量/曝光/点击 ============ def parse_num(s): if not s: return 0 s = str(s).replace("+", "").strip() s2 = s.replace("～", "~") if "万" in s2 or "w" in s2.lower(): v = re.sub(r"[^0-9.]", "", s2.replace("万", "").replace("w", "").replace("W", "")) return int(float(v or 0) * 10000) if "~" in s2: nums = [float(p) for p in re.findall(r"[\d.]+", s2)] if len(nums) >= 2: return int((nums[0] + nums[1]) / 2) return int(nums[0]) if nums else 0 m = re.search(r"[\d.]+", s2) return int(float(m.group(0))) if m else 0 table_rows = [] table_fp = os.path.join(SCRAPE_DIR, "table.json") if os.path.exists(table_fp): with open(table_fp, "r", encoding="utf-8") as f: table_rows = json.load(f) sales_bands = collections.Counter() table_relevant = [] for r in table_rows: name = r.get("name", "") if not name: continue cat = _guess_cat(name) if cat not in GOOD_CATS: continue if not _not_excluded(name): continue sales_mid = parse_num(r.get("sales", "")) expo_mid = parse_num(r.get("exposure", r.get("expo", ""))) click_mid = parse_num(r.get("clicks", "")) if sales_mid >= 10000: sales_bands["1万+"] += 1 elif sales_mid >= 1000: sales_bands["1000-1万"] += 1 elif sales_mid >= 100: sales_bands["100-1000"] += 1 else: sales_bands["100以下"] += 1 table_relevant.append({ "name": name, "cat": cat, "price": r.get("price", 0), "words": r.get("words", []), "sales": r.get("sales", ""), "sales_mid": sales_mid, "exposure": r.get("exposure", r.get("expo", "")), "exposure_mid": expo_mid, "clicks": r.get("clicks", ""), "clicks_mid": click_mid, "ctr": (round(click_mid / expo_mid * 100, 1) if expo_mid else 0), }) table_relevant.sort(key=lambda x: -x["sales_mid"]) high_sales_top = table_relevant[:10] for _r in table_rows: full = _r.get("name", "") if not full: continue pack_stat[parse_pack(full)] += 1 weight_stat[parse_weight_band(full)] += 1 usage_stat[parse_usage(full)] += 1 ctr_avg = round(sum(r["ctr"] for r in table_relevant if r["ctr"]) / max(len([r for r in table_relevant if r["ctr"]]), 1), 1) table_total = len(table_rows) table_rel_n = len(table_relevant) log("table: total=" + str(table_total) + " relevant=" + str(table_rel_n)) # D. 曝光/点击分组（机会信号） def _group_expo_click(rows): g_hi = [] # 高曝光低点击：主图/标题短板 → 上新抢流量机会 g_lo = [] # 低曝光高点击：产品吸引力强 → 值得加推 for r in rows: if not r["exposure_mid"] or r["exposure_mid"] < 500: continue if r["ctr"] <= 0: continue if r["exposure_mid"] >= 10000 and r["ctr"] < 20: g_hi.append(r) elif r["exposure_mid"] < 3000 and r["ctr"] >= 40: g_lo.append(r) g_hi.sort(key=lambda x: -x["exposure_mid"]) g_lo.sort(key=lambda x: -x["ctr"]) return [{"name": x["name"], "exposure": x["exposure"], "exposure_mid": x["exposure_mid"], "clicks": x["clicks"], "clicks_mid": x["clicks_mid"], "ctr": x["ctr"], "price": x["price"], "cat": x["cat"]} for x in g_hi[:6]], \ [{"name": x["name"], "exposure": x["exposure"], "exposure_mid": x["exposure_mid"], "clicks": x["clicks"], "clicks_mid": x["clicks_mid"], "ctr": x["ctr"], "price": x["price"], "cat": x["cat"]} for x in g_lo[:6]] expo_hi_group, expo_lo_group = _group_expo_click(table_relevant) def _sim(a, b): if not a or not b: return 0 pa = {a[i:i+2] for i in range(len(a) - 1)} pb = {b[i:i+2] for i in range(len(b) - 1)} if not pa or not pb: return 0 inter = len(pa & pb) return inter / min(len(pa), len(pb)) def find_table_match(name): best, bs = None, 0.0 for r in table_relevant: s = _sim(name, r["name"]) if s > bs: best, bs = r, s return (best, bs) if bs >= 0.3 else (None, 0) # ============ 2.3 P1：买家评价高频词（沿用） ============ buyer_words = [] review_fp = os.path.join(SCRAPE_DIR, "reviews.json") review_meta = {"records": 0, "source": "", "collected_at": "沿用历史"} if os.path.exists(review_fp): with open(review_fp, "r", encoding="utf-8") as f: rv = json.load(f) review_meta["records"] = len(rv.get("detail_records", [])) review_meta["source"] = rv.get("source", "") merged = collections.Counter() for dr in rv.get("detail_records", []): for tg in dr.get("good_tags", []): merged[tg["word"]] += tg["count"] buyer_words = [{"word": w, "count": c} for w, c in merged.most_common(12)] # ============ 3. 热搜词（当日真实词频，2026-10-09 起不再使用历史硬编码菜单规模） ============ SEARCH_WORDS_RAW = ["萝卜", "青萝卜", "白萝卜", "辣椒水", "牛肉酱", "梅菜扣肉", "香菇", "韭菜花", "笋丝", "石磨", "沙姜", "莴笋", "韭菜", "菜心", "大青椒", "韭菜花酱", "外婆菜", "榨菜", "酸菜", "泡菜", "咸菜", "豆豉", "雪菜", "红油", "洋姜", "萝卜干", "酱菜", "橄榄菜", "贡菜", "剁椒", "辣椒酱", "黄豆酱", "甜面酱", "芝麻酱", "蒜蓉", "酱豆", "下饭菜"] def freq_of(name, goods_list): pieces = [name[i:i+2] for i in range(max(len(name) - 1, 0))] def hit(title): if name in title: return True return any(p in title for p in pieces if len(p) == 2) return sum(1 for g in goods_list if hit(g["name"])) RANK_TITLES = [ "乌江爆炒酸菜30g开味佐餐拌面下饭菜", "湖南特产正宗外婆菜半成品咸菜袋装整件腌菜下饭菜批发", "乌江榨菜香辣酸辣豇豆30g袋小袋便携开味咸菜下饭菜", "十八巷子重庆榨菜40g袋10袋脆口榨菜", "涪陵发货重庆涪陵榨菜52克10袋纯榨菜", "正宗湘西外婆菜袋装萝卜干农家下饭菜", "爽脆麻辣萝卜干高山香脆香辣萝卜下饭菜川味开味菜", "蓬盛香港橄榄菜450g下饭菜潮汕特产酱菜", "十八巷子重庆榨菜脆口榨菜40g袋装速食菜佐餐咸菜", "聪厨官方湘西外婆菜200g4袋正宗湖南特产下饭菜", "麻辣香锅酱香香锅60克调味料排骨虾干锅鸡调料", "笑厨新疆番茄酱家用300g袋装", "珠江桥牌豉汁排骨酱70g豉汁王调味酱", "辣妹子牛肉香菇酱180g香辣下饭菜罐装拌面酱", "亨氏番茄沙司披萨汉堡意大利面炸鸡薯条点蘸酱料200g", "亨氏原味香甜沙拉酱120g", "老磨盘二八酱芝麻酱花生酱火锅蘸料酱200g热干面拌面酱", "河南老式酱豆西瓜酱豆香辣五香河南特产下饭酱豆夹馍神器黄豆酱", "味翁麻辣豆豉500g豆豉调味料四川原味特辣香辣豆鼓下饭菜", "海鲸黄豆酱东北大酱黄豆酱蘸酱炸酱调味调料", ] def rank_hit(name): return 0 RELEVANT = {"牛肉酱", "辣椒水", "石磨", "韭菜花", "韭菜花酱", "白萝卜", "青萝卜", "萝卜", "笋丝", "沙姜", "香菇", "大青椒", "菜心", "莴笋", "韭菜", "外婆菜", "榨菜", "酸菜", "泡菜", "咸菜", "豆豉", "雪菜", "红油", "洋姜", "萝卜干", "酱菜", "橄榄菜", "贡菜", "剁椒", "辣椒酱", "黄豆酱", "甜面酱", "芝麻酱", "蒜蓉", "酱豆", "下饭菜"} def comp_tier_of(pool_size): if pool_size <= 1000: return "蓝海" if pool_size >= 1500: return "红海" return "稳定" hot_words = [] for w in SEARCH_WORDS_RAW: if re.search(KEYWORD_BLACK, w): log("kw_black_skip: " + w) continue freq = freq_of(w, goods_list) # 热度分仅用当日真实采集成分：词频（五大池标题命中）+ 类目相关度；历史硬编码的 # 菜单规模/榜单命中已按用户要求移除。采用对数压缩映射保证区分度（避免高词频全部封顶100 # 导致看似未排序）：score=10+90*log(freq+1)/log(max_freq+1)（按全部候选词最大词频归一） max_freq = max([freq_of(x, goods_list) for x in SEARCH_WORDS_RAW] or [1]) score = min(10 + round(90 * (math.log(freq + 1) / math.log(max_freq + 1))), 100) if freq > 0 else 10 if w in RELEVANT: score = min(score + 2, 100) if score >= 80: tier = "爆款词" elif score >= 60: tier = "潜力词" else: tier = "常规词" # 相对竞争度（真实词频口径）：词频=类目池中真实命中数；高频=竞争激烈，低频=相对蓝海 comp_tier = "高竞争" if freq >= 30 else ("中竞争" if freq >= 12 else "低竞争") blue = (freq >= 8 and freq < 30) # 真实需求不低（≥8）且相对低频（<30）=相对蓝海 hot_words.append({"word": w, "freq": freq, "collected": 0, "comp_tier": comp_tier, "blue_ocean": blue, "score": score, "tier": tier}) hot_words.sort(key=lambda x: -x["score"]) blue_ocean_words = [w["word"] for w in hot_words if w["blue_ocean"]] # ============ 3.5b 分主题池词频（四大主题池：当季新品/全网热销/潜力商品/热门收藏，热门跟款已按用户2026-10-11指令移除） ============ # 每个主题池独立统计 SEARCH_WORDS_RAW 的标题命中词频，供看板菜单01按池切换查看 theme_freqs = {} for theme, _ in THEME_FILES: pool_goods = [g for g in goods_list if theme in g.get("themes", [])] tf = [] for w in SEARCH_WORDS_RAW: if re.search(KEYWORD_BLACK, w): continue f = sum(1 for g in pool_goods if w in g.get("name", "")) if f >= 1: tf.append({"word": w, "freq": f}) tf.sort(key=lambda x: -x["freq"]) theme_freqs[theme] = tf log("theme_freqs[%s]: %d 词" % (theme, len(tf))) # ============ 3.5 消费者正在搜·站内热搜/站内缺货词表（7个二级菜单之一，2026-10-09 模板） ============ consumer_search = None try: sa_fp = os.path.join(SCRAPE_DIR, "search_analysis.json") if os.path.exists(sa_fp): with open(sa_fp, "r", encoding="utf-8") as f: sa = json.load(f) meta = sa.get("_meta", {}) consumer_search = { "source": "机会商品页·消费者正在搜（站内热搜/站内缺货）", "capture_date": meta.get("capture_date", ""), "capture_time": meta.get("capture_time", ""), "hot_count": len(sa.get("hot_words_clean", [])), "short_count": len(sa.get("short_words_clean", [])), "hot_related": sa.get("hot_related", []), "short_related": sa.get("short_related", []), "excluded": sa.get("excluded", []), } log("consumer_search: hot=%d short=%d related=%s" % ( consumer_search["hot_count"], consumer_search["short_count"], ",".join(consumer_search["hot_related"]))) else: log("consumer_search: MISSING search_analysis.json (跳过，无站内热搜/缺货数据)") except Exception as ex: log("consumer_search error: " + str(ex)) # ============ 3.6 站内热搜/站内缺货商品池（2026-10-09 用户模板：各采集1000+，过滤主营无关后构建） ============ # 采集来源：机会商品页·消费者正在搜·站内热搜（cs_hot_goods.json）/ 站内缺货（cs_short_goods.json）， # 均为 dict 格式（key=name|price，value含name/price/hot，顶层 _meta）。 CS_POOL_FILES = [ ("站内热搜池", "cs_hot_goods.json"), ("站内缺货池", "cs_short_goods.json"), ] cs_pools = {} for pool_name, fname in CS_POOL_FILES: fp = os.path.join(SCRAPE_DIR, fname) if not os.path.exists(fp): log("cs_pool MISSING: " + fname) continue try: with open(fp, "r", encoding="utf-8") as f: d = json.load(f) if not isinstance(d, dict): log("cs_pool BAD FORMAT (expect dict): " + fname) continue meta = d.get("_meta", {}) items = [v for k, v in d.items() if k != "_meta" and isinstance(v, dict)] rel, excl = [], [] for it in items: nm = (it.get("name") or "").strip() if not nm: continue if _guess_cat(nm) in GOOD_CATS and _not_excluded(nm): try: it["hot_val"] = int(str(it.get("hot", "")).replace("热度", "").strip() or 0) except Exception: it["hot_val"] = 0 rel.append(it) else: excl.append(nm) rel.sort(key=lambda x: -x.get("hot_val", 0)) cs_pools[pool_name] = { "raw": len(items), "capture_date": meta.get("capture_date", ""), "capture_time": meta.get("capture_time", ""), "theme": meta.get("theme", pool_name), "relevant": len(rel), "excluded_n": len(excl), "top_goods": [{"name": x["name"], "price": x["price"], "hot": x["hot"], "hot_val": x.get("hot_val", 0)} for x in rel[:20]], "sample_excluded": excl[:12], } log("cs_pool[%s]: raw=%d relevant=%d excluded=%d" % (pool_name, len(items), len(rel), len(excl))) except Exception as ex: log("cs_pool error %s: %s" % (fname, str(ex))) # C. 趋势环比：对比上一日 hot_words.json prev_hot_fp = os.path.join(DATA_DIR, "hot_words.json") prev_hot = {} if os.path.exists(prev_hot_fp): try: with open(prev_hot_fp, "r", encoding="utf-8") as f: ph = json.load(f) for w in ph.get("words", []): prev_hot[w["word"]] = w except Exception: prev_hot = {} trend_alerts = [] for w in hot_words: pw = prev_hot.get(w["word"]) w["trend"] = {"prev_score": None, "delta_score": None, "prev_tier": None, "note": ""} if pw: ds = (w["score"] - pw.get("score", w["score"])) if pw.get("score") is not None else None w["trend"]["prev_score"] = pw.get("score") w["trend"]["delta_score"] = ds w["trend"]["prev_tier"] = pw.get("tier") if w["blue_ocean"] and not pw.get("blue_ocean"): w["trend"]["note"] = "新增蓝海" trend_alerts.append(w["word"] + " 新增蓝海") elif w["comp_tier"] == "高竞争" and pw.get("comp_tier") != "高竞争": w["trend"]["note"] = "竞争升温" trend_alerts.append(w["word"] + " 升为高竞争") elif ds is not None and ds >= 8: w["trend"]["note"] = "热度上涨" # E. 蓝海词数量突变预警 prev_blue_n = sum(1 for w in prev_hot.values() if w.get("blue_ocean")) cur_blue_n = len(blue_ocean_words) blue_alert = "" if prev_blue_n and cur_blue_n < prev_blue_n * 0.7: blue_alert = "蓝海词数量较上日减少超过30%%（%d→%d），赛道竞争加剧" % (prev_blue_n, cur_blue_n) trend_alerts.append(blue_alert) # A. 关键词分词拆解（扩充蓝海候选：标题高频2字词，与蓝海池外新词） ALL_NAMES = " ".join(g["name"] for g in goods_list) seg_cand = collections.Counter() for n in goods_list: nm = n["name"] for i in range(len(nm) - 1): seg = nm[i:i+2] if re.fullmatch(r"[\u4e00-\u9fa5]{2}", seg) and seg not in ("一个", "产品", "批发", "包邮", "热销", "精选", "正宗", "特价"): seg_cand[seg] += 1 seg_top = [{"word": w, "freq": c} for w, c in seg_cand.most_common(30)] # 二级词蓝海判定：出现频次>=5 且不在红海菜单词内（池数据未知按无池处理） RED_WORDS = {"萝卜", "牛肉酱", "香菇", "排骨", "笋丝", "石磨"} # 无关/无意义分词碎片黑名单（跨词边界碎片、泛用词、非目标类目词，用户要求清除） SEG_BLACK = {"菜下", "菜菜", "菜泡", "菜特", "菜榨", "菜酱", "生油", "食用", "速食", "包邮", "特价", "精选", "一个", "产品", "批发", "热销", "正宗"} seg_blue = [{"word": s["word"], "freq": s["freq"]} for s in seg_top if s["word"] not in RED_WORDS and s["word"] not in set(blue_ocean_words) and s["word"] not in SEG_BLACK and s["freq"] >= 5] SCENE_WORDS = ["下饭", "拌面", "拌饭", "即食", "开胃", "早餐", "整箱", "特产", "批发", "老坛", "香辣", "麻辣", "酸辣", "脆爽", "脆", "泡椒", "剁椒", "豆豉", "榨菜", "咸菜", "梅菜", "萝卜", "贡菜", "酸菜", "外婆菜", "辣椒", "牛肉", "香菇", "夹馍", "蘸", "无添加", "低盐", "家庭装", "组合装", "瓶装", "袋装", "湖南特产", "川味", "重庆", "潮汕", "河南"] scene_list = [] for w in SCENE_WORDS: f = freq_of(w, goods_list) if f > 0: scene_list.append({"word": w, "freq": f, "score": min(10 + f * 6, 100), "tier": "爆款词" if f >= 20 else ("潜力词" if f >= 10 else "常规词")}) scene_list.sort(key=lambda x: -x["freq"]) # ============ 3.5 竞品关键词池（B） ============ def load_kw_pool(fname): fp = os.path.join(SCRAPE_DIR, fname) if not os.path.exists(fp): return None with open(fp, "r", encoding="utf-8") as f: d = json.load(f) if not isinstance(d, dict): return None meta = d.get("_meta", {}) items = [] for k, v in d.items(): if k == "_meta" or not isinstance(v, dict): continue items.append(v) return {"keyword": meta.get("keyword", fname), "count": len(items), "capture_time": meta.get("capture_time", ""), "limited": meta.get("limited", False), "items": items} kw_pools = [] if not SKIP_KW_POOLS: for kw in COMPETITOR_KWS: fn = os.path.join(SCRAPE_DIR, "kw_%s.json" % kw) pool = load_kw_pool(os.path.basename(fn)) if os.path.exists(fn) else None if not pool: log("kw_pool MISSING: " + kw) continue items = pool["items"] rel = [g for g in items if _guess_cat(g.get("name", "")) in GOOD_CATS] rel = [g for g in rel if _not_excluded(g.get("name", ""))] def _hv(g): try: return int(str(g.get("hot", "")).replace("热度", "").strip() or 0) except Exception: return 0 for g in items: g["_hv"] = _hv(g) top = sorted(rel, key=lambda x: -x["_hv"])[:8] kw_pools.append({ "keyword": pool["keyword"], "total_collected": pool["count"], "relevant": len(rel), "capture_time": pool["capture_time"], "limited": pool.get("limited", False), "top_goods": [{"name": g.get("name", ""), "price": g.get("price", ""), "hot": g.get("hot", ""), "tags": g.get("tags", [])} for g in top], }) log("kw_pools: " + json.dumps([(p["keyword"], p["total_collected"], p["relevant"]) for p in kw_pools], ensure_ascii=False)) # ============ 4. 榜单（F. 时效标注） ============ rank_sauce = [ {"rank": 1, "name": "麻辣香锅酱香香锅60克调味料", "index": 6324, "sold": "5000+件"}, {"rank": 2, "name": "笑厨新疆番茄酱家用300g袋装", "index": 6037, "sold": "5000+件"}, {"rank": 3, "name": "珠江桥牌豉汁排骨酱70g", "index": 5774, "sold": "5000+件"}, {"rank": 4, "name": "辣妹子牛肉香菇酱180g", "index": 5534, "sold": "5000+件"}, {"rank": 5, "name": "亨氏番茄沙司200g", "index": 5312, "sold": "5000+件"}, {"rank": 6, "name": "亨氏原味香甜沙拉酱120g", "index": 5108, "sold": "5000+件"}, {"rank": 7, "name": "老磨盘二八酱芝麻花生酱200g", "index": 4919, "sold": "5000+件"}, {"rank": 8, "name": "河南老式酱豆西瓜酱豆", "index": 4743, "sold": "5000+件"}, {"rank": 9, "name": "味翁麻辣豆豉500g", "index": 4580, "sold": "5000+件"}, {"rank": 10, "name": "海鲸黄豆酱东北大酱", "index": 4427, "sold": "5000+件"}, ] rank_fast = [ {"rank": 1, "name": "乌江爆炒酸菜30g", "index": 6388, "sold": "5000+件"}, {"rank": 2, "name": "湘西外婆菜半成品咸菜袋装", "index": 6098, "sold": "5000+件"}, {"rank": 3, "name": "乌江榨菜香辣酸辣豇豆30g", "index": 5833, "sold": "5000+件"}, {"rank": 4, "name": "十八巷子重庆榨菜40g", "index": 5590, "sold": "5000+件"}, {"rank": 5, "name": "涪陵榨菜52克10袋", "index": 5366, "sold": "5000+件"}, {"rank": 6, "name": "湘西外婆菜萝卜干农家下饭菜", "index": 5160, "sold": "5000+件"}, {"rank": 7, "name": "爽脆麻辣萝卜干高山香脆", "index": 4968, "sold": "5000+件"}, {"rank": 8, "name": "蓬盛橄榄菜450g潮汕特产", "index": 4791, "sold": "5000+件"}, {"rank": 9, "name": "重庆榨菜脆口榨菜40g袋装", "index": 4626, "sold": "5000+件"}, {"rank": 10, "name": "聪厨湘西外婆菜200g", "index": 4472, "sold": "5000+件"}, ] # ============ 5. 推荐上架新品（热度+销量信号+竞争度+主题+新品） ============ JUNK_KEYS = ["种子", "秧苗", "苗", "盆栽", "肥料", "农药", "饲料", "农资", "活体", "宠物", "化肥", "营养土", "花盆", "农用", "大棚", "蔬菜种", "菌包", "虾苗", "鱼苗", "鸡苗", "鸭苗", "菜苗", "育苗", "秧"] def build_recs(): cands = [] for g in goods_list: name = g.get("name", "") if any(k in name for k in JUNK_KEYS): continue themes = g.get("themes", []) if not themes: continue is_season = "当季新品" in g.get("tags", []) hot_v = g.get("hot_val", 0) p = g["price"] w = 0 if is_season and 3 <= p <= 30: w = 5 elif hot_v >= 100 and 3 <= p <= 30 and len(themes) >= 2: w = 3 elif 3 <= p <= 15 and len(themes) >= 2: w = 1 if w == 0: continue trow, ts = find_table_match(name) sales_signal = 0 if trow and ts >= 0.35: sales_signal = 1 if trow["sales_mid"] >= 1000 else 2 if trow["sales_mid"] >= 10000 else 0 w += sales_signal blue_hits = [bw for bw in blue_ocean_words if bw in name] if blue_hits: w += 1 cands.append((g, w, sales_signal, blue_hits, trow)) cands.sort(key=lambda x: (-x[1], -x[0].get("hot_val", 0), x[0]["price"])) seen_cat = collections.Counter() recs = [] for g, w, ss, blue_hits, trow in cands: if len(recs) >= 12: break if seen_cat[g["cat"]] >= 3: continue seen_cat[g["cat"]] += 1 rec = { "name": g["name"], "price": g["price"], "cat": g["cat"], "tags": g.get("tags", []), "themes": g.get("themes", []), "hot_val": g.get("hot_val", 0), "weight": w, "blue_words": blue_hits, } if trow: rec["sales_signal"] = trow["sales_mid"] rec["sales_txt"] = trow["sales"] rec["ctr"] = trow["ctr"] rec["pack"] = parse_pack(trow["name"]) rec["usage"] = parse_usage(trow["name"]) rec["weight_band"] = parse_weight_band(trow["name"]) else: rec["sales_signal"] = 0 rec["sales_txt"] = "无销量数据" rec["pack"] = g.get("pack", "其他") rec["usage"] = g.get("usage", "通用") rec["weight_band"] = g.get("weight_band", "") recs.append(rec) return recs recs = build_recs() # ============ 6. 行业趋势 ============ industry = { "source": "豆丁网《2026年中国酱菜腌菜行业市场规模及投资前景预测分析报告》等", "douyin_plays": "42.8亿次(2025年抖音酱腌菜话题播放量)", "kw_growth": "「无添加」「脆爽」「下饭神器」年搜索增长率均超90%", "low_salt_trend": "低盐/减盐/低钠、零添加、有机成为核心升级方向", "olive_veg_boom": "潮汕橄榄菜搜索浏览量涨幅327%(2026-05 得物数据)", } # ============ 写出 ============ def dump(fname, obj): with open(os.path.join(DATA_DIR, fname), "w", encoding="utf-8") as f: json.dump(obj, f, ensure_ascii=False, indent=2) menus_info = [{"word": w, "count": MENU_COUNTS.get(w, 0), "collected": 0} for w in SEARCH_WORDS_RAW if w in MENU_COUNTS and not re.search(KEYWORD_BLACK, w)] theme_info = [{"word": t, "count": theme_collected.get(t, 0), "collected": theme_collected.get(t, 0)} for t, _ in THEME_FILES] dump("opportunity_goods.json", { "collected_at": now_str, "source": "拼多多商家后台-机会商品页·热门主题四大池（当季新品/全网热销/潜力商品/热门收藏，热门跟款按用户2026-10-11指令已停采）+ 前台商品评价高频词(P1沿用)", "total": len(goods_list), "raw_records": raw_records, "season_new": season_new, "themes": theme_info, "menus": menus_info, "consumer_search": consumer_search, "cs_pools": cs_pools, "recs": recs, "goods": sorted(goods_list, key=lambda x: x["price"]), "cat_stat": dict(cat_stat.most_common()), "price_bands": dict(price_bands), "spec_stat": {"pack": dict(pack_stat.most_common()), "weight": dict(weight_stat.most_common()), "usage": dict(usage_stat.most_common())}, "attr_stat": [{"word": w, "count": c} for w, c in attr_top], "buyer_words": buyer_words, "sales_sample": { "table_total": table_total, "relevant": table_rel_n, "high_sales_top": high_sales_top, "sales_bands": dict(sales_bands), "ctr_avg": ctr_avg, }, "expo_click_groups": {"hi_expo_lo_ctr": expo_hi_group, "lo_expo_hi_ctr": expo_lo_group}, "blue_ocean_words": blue_ocean_words, "seg_words_top": seg_top, "seg_blue_candidates": seg_blue, "kw_pools": kw_pools, "trend_alerts": trend_alerts, "review_meta": review_meta, }) dump("hot_words.json", { "collected_at": now_str, "source": "拼多多机会商品页热门主题五大池标题词频 + 热搜词菜单官方规模(竞争池) + 消费者正在搜·站内热搜/站内缺货 + 拼多多官方排行榜 + 上日环比", "method": "热度分=10+90*log(词频+1)/log(最高词频+1)，封顶100；词频=关键词2字片段命中五大主题池机会商品标题数；竞争度=当日真实词频分档；trend=与上一日对比", "words": hot_words, "theme_freqs": theme_freqs, "scene_words": scene_list, "consumer_search": consumer_search, }) dump("rank_lists.json", { "captured_at": now_str, "source": "拼多多官方排行榜 m.pinduoduo.com（本次未能实时定位榜单入口，沿用最近采集数据，热卖指数为历史参考值）", "sauce": rank_sauce, "fast_food": rank_fast, }) dump("store_stats.json", { "industry": industry, "generated_at": now_str, }) log("done. merged_goods: %d raw_records: %d season_new: %d" % (len(goods_list), raw_records, season_new)) log("themes: " + json.dumps(theme_info, ensure_ascii=False)) log("cat_stat: " + json.dumps(dict(cat_stat.most_common()), ensure_ascii=False)) log("price_bands: " + json.dumps(dict(price_bands))) log("spec pack: " + json.dumps(dict(pack_stat.most_common()), ensure_ascii=False)) log("spec usage: " + json.dumps(dict(usage_stat.most_common()), ensure_ascii=False)) log("table relevant: %d ctr_avg: %s" % (table_rel_n, ctr_avg)) log("blue_ocean: " + json.dumps(blue_ocean_words, ensure_ascii=False)) log("seg_blue_candidates: " + json.dumps(seg_blue, ensure_ascii=False)) log("expo_hi: " + json.dumps([x["name"][:12] for x in expo_hi_group], ensure_ascii=False)) log("expo_lo: " + json.dumps([x["name"][:12] for x in expo_lo_group], ensure_ascii=False)) log("trend_alerts: " + json.dumps(trend_alerts, ensure_ascii=False)) log("buyer_words: " + json.dumps([(x["word"], x["count"]) for x in buyer_words[:8]], ensure_ascii=False)) log("hot_words_top: " + json.dumps([(x["word"], x["score"], x["freq"], x["comp_tier"]) for x in hot_words[:10]], ensure_ascii=False)) log("recs_top: " + json.dumps([(x["name"][:20], x["weight"], x.get("sales_signal", 0)) for x in recs[:6]], ensure_ascii=False))