[{"data":1,"prerenderedAt":402},["ShallowReactive",2],{"tool-backlink-analysis-common-crawl":3,"footer-total":373,"related-backlink-analysis-common-crawl":374},{"id":4,"title":5,"author":6,"bestFor":7,"body":8,"category":330,"cons":331,"cover":337,"date":338,"description":339,"draft":340,"extension":341,"meta":342,"navigation":343,"notFor":344,"path":345,"pricing":346,"pros":350,"region":356,"score":357,"scoreBreakdown":358,"seo":364,"stem":365,"tags":366,"toolName":201,"updated":338,"url":370,"vendor":371,"__hash__":372},"tools\u002Ftools\u002Fbacklink-analysis\u002Fcommon-crawl.md","Common Crawl 评测","元析评测","有数据工程团队、要做大规模链接图分析或训练自有指标的机构与 SaaS 公司",{"type":9,"value":10,"toc":302},"minimark",[11,15,23,26,29,45,48,52,55,58,62,65,68,72,75,78,81,84,87,90,94,97,101,104,108,111,115,118,122,125,128,184,187,190,265,271,275,279,282,286,289,293,296,299],[12,13,14],"h2",{"id":14},"结论先行",[16,17,18,22],"p",{},[19,20,21],"strong",{},"推荐给工程团队,总分 7.0,数据完全免费,2026-11 核对时无任何付费门槛。"," 它适合有数据工程能力、要做大规模链接图分析或者训练自有权威度指标的机构,不适合想点开网页就查反链的人。",[16,24,25],{},"必须先说清楚:它不是反链工具,是原材料。你要的\"某个域名的反链列表\",得自己从 CDX 索引和 Web Graph 里算出来,写代码、跑集群、调口径,少一样都出不来结果。把它当工具买会失望,当数据源用才有价值。",[12,27,28],{"id":28},"这篇评测基于什么数据",[30,31,32,36,39,42],"ul",{},[33,34,35],"li",{},"官网首页:2026-11 打开 commoncrawl.org,核对了语料规模、时间跨度、月度增量与非营利机构属性。",[33,37,38],{},"数据形态:官网列出 CDXJ 索引、URL 索引、Web Graphs、最新批次、抓取统计与勘误表六个数据入口,均为公开可下载。",[33,40,41],{},"机构背景:页面写明该基金会成立于 2007 年,为 501(c)(3) 非营利组织,语料被超过一万篇研究论文引用。",[33,43,44],{},"未实测声明:我们未下载批次数据、未在云上跑过反链 join;下文对计算成本与流程的描述基于官方文档与公开技术资料,不含虚构的实测数字。",[12,46,47],{"id":47},"核心功能",[49,50,51],"h3",{"id":51},"三百亿页的公开语料",[16,53,54],{},"规模是它唯一无法被替代的理由。3000 亿页、跨 15 年、每月新增 30 亿到 50 亿页,任何商业反链工具都不会把原始网页免费给你,而它给的是 WARC 文件本身——页面正文、响应头、链接结构全在里面。",[16,56,57],{},"每月一次抓取,批次按年份与周编号命名。这意味着你可以拿 2019 年的批次和 2026 年的批次对比同一个域名,看它什么时候开始被大量引用、什么时候链接消失。这种时间维度,商业工具的免费档基本不给。",[49,59,61],{"id":60},"cdx-索引与-url-索引","CDX 索引与 URL 索引",[16,63,64],{},"CDX 索引解决\"某个 URL 有没有被抓过、抓了几次、存在哪个文件里\"。它是一条条记录,包含抓取时间、状态码、文件偏移量。要查一个域名的历史快照,先查 CDX,再去对应 WARC 文件里定位读取。",[16,66,67],{},"URL 索引是另一层:按 URL 排序的倒排结构,便于批量扫描某个后缀或某个路径下的全部页面。做站群分析时,先按索引圈定范围,再回 WARC 取正文,是标准流程。",[49,69,71],{"id":70},"web-graph反链关系的来源","Web Graph:反链关系的来源",[16,73,74],{},"反链分析真正要用的是 Web Graph。它把每轮抓取里的页面当作节点、链接当作边,输出成图数据,分为顶点表与边表。有了边表,\"谁链向 example.com\"就是一次过滤加聚合。",[16,76,77],{},"难点在规模:边表是千亿级别,必须用 Athena、BigQuery 或 Spark 这类分布式引擎跑,单机 SQLite 根本装不下。想拿到\"前 100 条反链\"这种小结果,前面要先扫过巨量数据。",[49,79,80],{"id":80},"抓取规则与勘误",[16,82,83],{},"它公开 CCBot 的抓取规则,站长可以通过 robots.txt 或它的退出登记表要求不被抓取;官网还维护勘误表,记录某些批次的数据缺陷。",[16,85,86],{},"这两项对企业用户重要:如果你的站排除了 CCBot,你在这份语料里就是缺失的,基于它做的分析会有系统性偏差。用之前先确认自己的站有没有被排除。",[12,88,89],{"id":89},"评分明细",[49,91,93],{"id":92},"功能完整度7810","功能完整度:7.8\u002F10",[16,95,96],{},"作为数据源,它的入口很全:原始文件、索引、图数据、统计、勘误都有,研究需要的材料基本齐备。扣分在于它刻意不做产品化——没有查询界面、没有 API 封装、没有现成的锚文本聚合,所有加工都得自己做。",[49,98,100],{"id":99},"易用性4610","易用性:4.6\u002F10",[16,102,103],{},"这是它最劝退的一项,而且是设计使然。第一次用的人要面对 WARC 格式、CDX 记录格式、S3 路径规则、分布式查询四道门槛,官方文档写得清楚,但假设你懂这些。没有工程背景的人,建议直接放弃。",[49,105,107],{"id":106},"数据质量8010","数据质量:8.0\u002F10",[16,109,110],{},"覆盖面广、批次留档完整、口径公开,是学术研究的常用基准。弱点也明确:抓取不按流行度排序,长尾站的链接常常抓不到;页面是快照,不保证与线上一致;不同批次的覆盖率波动会让时间序列出现假拐点。",[49,112,114],{"id":113},"性价比8610","性价比:8.6\u002F10",[16,116,117],{},"数据本身零成本,这是它最锋利的地方。但\"免费\"只指数据:PB 级扫描要用云资源,存储、计算、流量三项加起来,一次全量分析的费用可能超过一年商业工具订阅。给小团队算账时,别只看授权费。",[49,119,121],{"id":120},"客户支持5410","客户支持:5.4\u002F10",[16,123,124],{},"有邮件列表、Discord 社区、FAQ 与技术文档,研究者之间互相解答的氛围不错。但没有工单、没有 SLA、没有商业支持;非营利机构的资源有限,基础设施偶发中断时只能等。",[12,126,127],{"id":127},"定价说明",[129,130,131,147],"table",{},[132,133,134],"thead",{},[135,136,137,141,144],"tr",{},[138,139,140],"th",{},"套餐",[138,142,143],{},"价格",[138,145,146],{},"说明",[148,149,150,162,173],"tbody",{},[135,151,152,156,159],{},[153,154,155],"td",{},"语料下载",[153,157,158],{},"$0",[153,160,161],{},"WARC\u002FWAT\u002FWET 与索引公开,无授权费",[135,163,164,167,170],{},[153,165,166],{},"计算与存储",[153,168,169],{},"按云厂商计价",[153,171,172],{},"自建查询的集群、对象存储与流量费用",[135,174,175,178,181],{},[153,176,177],{},"商业支持",[153,179,180],{},"不提供",[153,182,183],{},"无 SLA、无工单,仅社区与邮件列表",[16,185,186],{},"这张表要说清一件事:免费的是数据,不是分析。一次针对单个域名的反链查询,用商业工具每月几十美元;用 Common Crawl 自建,得先付集群钱、再付工程师时间。什么规模下自建划算,取决于你一年要跑多少次。",[12,188,189],{"id":189},"和竞品对比",[129,191,192,208],{},[132,193,194],{},[135,195,196,199,202,205],{},[138,197,198],{},"维度",[138,200,201],{},"Common Crawl",[138,203,204],{},"Ahrefs API",[138,206,207],{},"Moz API",[148,209,210,223,237,251],{},[135,211,212,215,218,221],{},[153,213,214],{},"起步价",[153,216,217],{},"免费(数据)",[153,219,220],{},"按调用量计费,需订阅",[153,222,220],{},[135,224,225,228,231,234],{},[153,226,227],{},"交付形态",[153,229,230],{},"原始 WARC + 图数据",[153,232,233],{},"现成反链与指标接口",[153,235,236],{},"现成反链与权威度接口",[135,238,239,242,245,248],{},[153,240,241],{},"强项",[153,243,244],{},"规模最大、可回溯 15 年",[153,246,247],{},"索引新鲜、字段成熟",[153,249,250],{},"权威度指标被行业广泛引用",[135,252,253,256,259,262],{},[153,254,255],{},"短板",[153,257,258],{},"需自建计算、无产品界面",[153,260,261],{},"按量计费,大规模调用贵",[153,263,264],{},"索引规模小于前者",[16,266,267,270],{},[19,268,269],{},"怎么选",":一年只查几百个域名,买 API 或订阅,别自建;要做全站群链接图、要训练自己的评分模型、要回溯五年前的数据,才值得上 Common Crawl。中间态的做法是先用它做样本验证思路,再决定要不要为生产环境付费。",[12,272,274],{"id":273},"faq","FAQ",[49,276,278],{"id":277},"q-没有数据团队能用它查反链吗","Q: 没有数据团队,能用它查反链吗?",[16,280,281],{},"A: 不建议。最少要会三件事:从 S3 读 WARC 文件、写分布式 SQL 或 Spark 任务、把结果落成可查的表。缺任何一项,你都会卡在第一步——连数据长什么样都看不到。想低成本入门,先用商业工具的免费额度验证需求。",[49,283,285],{"id":284},"q-它能直接告诉我某个域名的反链吗","Q: 它能直接告诉我某个域名的反链吗?",[16,287,288],{},"A: 不能直接给,但能算出来。路径是:用 Web Graph 的边表筛出指向该域名的边,拿到来源 URL,再回 CDX 与 WARC 取锚文本和页面元信息。全流程自己实现,官方只提供原料与文档。",[49,290,292],{"id":291},"q-免费数据为什么还要算成本","Q: 免费数据,为什么还要算成本?",[16,294,295],{},"A: 因为数据量太大。PB 级的边表扫一遍,云上的计算与流量费用是实打实的支出;再加上工程人力,一次全量分析的投入通常高于一年商业订阅。它的省钱前提是\"你会用、且用得多次\"。",[12,297,298],{"id":298},"评测方法说明",[16,300,301],{},"元析评测围绕功能完整度、易用性、数据质量、性价比、客户支持五个维度打分,每项满分 10 分,加权折算总分。文中语料规模、时间跨度、月度增量、机构属性与数据入口均于 2026-11 对照 Common Crawl 官网核实;计算成本部分为公开资料推算,不含虚构的实测账单。我们未做云上实测。",{"title":303,"searchDepth":304,"depth":304,"links":305},"",2,[306,307,308,315,322,323,324,329],{"id":14,"depth":304,"text":14},{"id":28,"depth":304,"text":28},{"id":47,"depth":304,"text":47,"children":309},[310,312,313,314],{"id":51,"depth":311,"text":51},3,{"id":60,"depth":311,"text":61},{"id":70,"depth":311,"text":71},{"id":80,"depth":311,"text":80},{"id":89,"depth":304,"text":89,"children":316},[317,318,319,320,321],{"id":92,"depth":311,"text":93},{"id":99,"depth":311,"text":100},{"id":106,"depth":311,"text":107},{"id":113,"depth":311,"text":114},{"id":120,"depth":311,"text":121},{"id":127,"depth":304,"text":127},{"id":189,"depth":304,"text":189},{"id":273,"depth":304,"text":274,"children":325},[326,327,328],{"id":277,"depth":311,"text":278},{"id":284,"depth":311,"text":285},{"id":291,"depth":311,"text":292},{"id":298,"depth":304,"text":298},"backlink-analysis",[332,333,334,335,336],"没有界面,没有查询框,一行 SQL 都不会写的人无法使用","自建反链查询要处理 PB 级数据,单机跑不动,必须上云做分布式计算","爬虫不对全网做优先级排序,长尾站的覆盖率远低于商业索引","没有现成的锚文本聚合、权威度评分、垃圾判定,全部要自己实现","非营利机构,无工单与 SLA,基础设施靠赞助,偶发中断只能等",null,"2026-11-16","Common Crawl 评测:免费开放的网页语料库,3000 亿页、15 年积累、每月新增 30-50 亿页。不提供现成反链报告,要自己用 CDX 索引和 Web Graph 做关联。适合有数据工程能力的团队,个人站长不必碰。",false,"md",{},true,"想点开网页就查反链的 SEO 从业者,以及没有云端计算预算的小团队","\u002Ftools\u002Fbacklink-analysis\u002Fcommon-crawl",{"free":343,"startingPrice":303,"currency":347,"billingPeriod":348,"notes":349},"免费","一次性","数据免费开放,2026-11 核对。以 WARC\u002FWAT\u002FWET 文件与 CDX 索引形式发布,自建反链查询所需的存储、计算与流量费用由使用方承担;该机构为非营利基金会,无商业套餐、无 SLA。",[351,352,353,354,355],"3000 亿页、15 年积累的公开语料,规模超过任何单家商业反链索引","每月新增 30 亿到 50 亿页,历史批次留档可回溯,能算时间序列","提供 CDX 索引、URL 索引与 Web Graph 三类入口,反链关系可从图数据里推","数据以 WARC 格式裸存,锚文本、页面正文、响应头都能自己取,不受工具口径限制","完全免费,非营利基金会运营,附抓取统计与勘误表,口径可追溯","global",7,{"features":359,"ease":360,"data":361,"price":362,"support":363},7.8,4.6,8,8.6,5.4,{"title":5,"description":339},"tools\u002Fbacklink-analysis\u002Fcommon-crawl",[367,368,369,201],"外链分析","开放数据","Web Graph","https:\u002F\u002Fcommoncrawl.org\u002F","Common Crawl Foundation","E8sZM2OKKRhmgjAXJDzsGe6Xll41-FjJUahgvx2a5Cw",1016,[375,385,394],{"path":376,"toolName":377,"description":378,"score":362,"pricing":379,"vendor":384,"date":338,"category":330},"\u002Ftools\u002Fbacklink-analysis\u002Fahrefs-backlink-checker","Ahrefs Backlink Checker","Ahrefs Backlink Checker 评测:免费查任意域名前 100 条外链,付费套餐 $29\u002F月起。按 DR、锚文本和引用域拆解竞品链接结构很顺手,但免费版不给导出、不含历史数据,做百度外链分析基本无效。",{"free":343,"startingPrice":380,"currency":381,"billingPeriod":382,"notes":383},"29","USD","月","免费 Backlink Checker 不登录即可查任意域名前 100 条外链,不提供导出与历史数据;付费套餐 Starter $29\u002F月(仅月付,200 积分\u002F月)、Lite $129\u002F月、Standard $249\u002F月、Advanced $449\u002F月,年付按 10 个月计价。价格于 2026-11 核对官网。","Ahrefs",{"path":386,"toolName":387,"description":388,"score":389,"pricing":390,"vendor":393,"date":338,"category":330},"\u002Ftools\u002Fbacklink-analysis\u002Fsemrush-backlink-analytics","Semrush Backlink Analytics","Semrush Backlink Analytics 评测:反链总量、毒性评分、新增与流失放在同一屏看,做外链体检比单查清单省事。订阅 $139\u002F月起,单买反链模块不划算,适合已经在用 Semrush 做关键词的团队顺手做链接审计。",8.4,{"free":340,"startingPrice":391,"currency":381,"billingPeriod":382,"notes":392},"139","2026 年 5 月改版后不再售卖旧版 Pro\u002FGuru\u002FBusiness 三档,现行 SEO 套餐 $139\u002F月、Starter 套餐 $199\u002F月,更高档按站点数与关键词量递增;年付约省 15%。免费注册账号每日可发 10 次请求,仅够看摘要。价格于 2026-11 核对官网。","Semrush",{"path":395,"toolName":396,"description":397,"score":398,"pricing":399,"vendor":396,"date":338,"category":330},"\u002Ftools\u002Fbacklink-analysis\u002Fpitchbox","Pitchbox","Pitchbox 评测:外联自动化平台,Pro $300\u002F月(年付 $210\u002F月)含 2 用户、2,000 封外联信、600 万数据额度,Advanced $600\u002F月放开用户数到无限。适合代理公司批量投放,单人博客买它纯属浪费。",8.2,{"free":340,"startingPrice":400,"currency":381,"billingPeriod":382,"notes":401},"300","Pro $300\u002F月(年付 $210\u002F月,2 用户、2 个发信邮箱、每月 2,000 封外联信、25 个活动、5 万联系人、每年 60 万数据额度);Advanced $600\u002F月(年付 $420\u002F月,用户不限、6 个发信邮箱、每月 5,000 封、25 个工作区、20 万联系人、数据额度不限)。年付约省 30%。更高档位需联系销售。价格于 2026-11 核对官网。",1794788218321]