[{"data":1,"prerenderedAt":667},["ShallowReactive",2],{"navigation_docs":3,"-guide-custom":68,"-guide-custom-surround":662},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":28,"body":70,"description":654,"extension":655,"links":656,"meta":657,"navigation":169,"path":29,"seo":659,"stem":30,"__hash__":661},"docs\u002F1.guide\u002F6.custom.md",{"type":71,"value":72,"toc":649},"minimark",[73,77,545,550,601,617,624,638,645],[74,75,76],"p",{},"Found another passive index? It takes one class and one call.",[78,79,85],"pre",{"className":80,"code":81,"filename":82,"language":83,"meta":84,"style":84},"language-ts shiki shiki-themes urls urls urls","import {\n  create,\n  Provider,\n  register,\n  resolveDomain,\n  UrlCollector,\n  type DiscoveredUrl,\n  type DiscoverOptions,\n} from \"@agntn\u002Furls\";\n\nclass MyIndex extends Provider {\n  static readonly key = \"myindex\";\n\n  get capabilities() {\n    return { discover: true };\n  }\n\n  async discover(domain: string, options?: DiscoverOptions): Promise\u003CDiscoveredUrl[]> {\n    const target = resolveDomain(domain, this.name);\n    const collector = new UrlCollector(options, domain);\n    const api = `https:\u002F\u002Findex.example\u002Furls?domain=${target}`;\n    const rows = await this.getJSON\u003C{ url: string; seen?: string }[]>(api, { signal: options?.signal });\n\n    for (const row of rows) {\n      if (collector.done) break;\n      collector.push(this.name, row.url, api, row.seen);\n    }\n    return collector.results;\n  }\n}\n\nregister(MyIndex, {\n  key: \"myindex\",\n  capabilities: { discover: true },\n  load: () => Promise.resolve(MyIndex),\n});\n\nconst mine = await create(\"myindex\");\n","my-index.ts","ts","",[86,87,88,101,107,113,119,125,131,140,148,164,171,187,207,212,224,233,239,244,288,305,324,345,381,386,407,421,433,439,447,452,458,463,472,484,490,511,517,522],"code",{"__ignoreMap":84},[89,90,93,97],"span",{"class":91,"line":92},"line",1,[89,94,96],{"class":95},"skH_V","import",[89,98,100],{"class":99},"s38Sx"," {\n",[89,102,104],{"class":91,"line":103},2,[89,105,106],{"class":99},"  create,\n",[89,108,110],{"class":91,"line":109},3,[89,111,112],{"class":99},"  Provider,\n",[89,114,116],{"class":91,"line":115},4,[89,117,118],{"class":99},"  register,\n",[89,120,122],{"class":91,"line":121},5,[89,123,124],{"class":99},"  resolveDomain,\n",[89,126,128],{"class":91,"line":127},6,[89,129,130],{"class":99},"  UrlCollector,\n",[89,132,134,137],{"class":91,"line":133},7,[89,135,136],{"class":95},"  type",[89,138,139],{"class":99}," DiscoveredUrl,\n",[89,141,143,145],{"class":91,"line":142},8,[89,144,136],{"class":95},[89,146,147],{"class":99}," DiscoverOptions,\n",[89,149,151,154,157,161],{"class":91,"line":150},9,[89,152,153],{"class":99},"} ",[89,155,156],{"class":95},"from",[89,158,160],{"class":159},"shU9J"," \"@agntn\u002Furls\"",[89,162,163],{"class":99},";\n",[89,165,167],{"class":91,"line":166},10,[89,168,170],{"emptyLinePlaceholder":169},true,"\n",[89,172,174,177,181,184],{"class":91,"line":173},11,[89,175,176],{"class":95},"class",[89,178,180],{"class":179},"sK71F"," MyIndex",[89,182,183],{"class":95}," extends",[89,185,186],{"class":99}," Provider {\n",[89,188,190,193,196,199,202,205],{"class":91,"line":189},12,[89,191,192],{"class":95},"  static",[89,194,195],{"class":95}," readonly",[89,197,198],{"class":99}," key ",[89,200,201],{"class":95},"=",[89,203,204],{"class":159}," \"myindex\"",[89,206,163],{"class":99},[89,208,210],{"class":91,"line":209},13,[89,211,170],{"emptyLinePlaceholder":169},[89,213,215,218,221],{"class":91,"line":214},14,[89,216,217],{"class":95},"  get",[89,219,220],{"class":179}," capabilities",[89,222,223],{"class":99},"() {\n",[89,225,227,230],{"class":91,"line":226},15,[89,228,229],{"class":95},"    return",[89,231,232],{"class":99}," { discover: true };\n",[89,234,236],{"class":91,"line":235},16,[89,237,238],{"class":99},"  }\n",[89,240,242],{"class":91,"line":241},17,[89,243,170],{"emptyLinePlaceholder":169},[89,245,247,250,253,256,259,262,265,268,271,274,276,279,282,285],{"class":91,"line":246},18,[89,248,249],{"class":95},"  async",[89,251,252],{"class":179}," discover",[89,254,255],{"class":99},"(domain",[89,257,258],{"class":95},":",[89,260,261],{"class":179}," string",[89,263,264],{"class":99},", options",[89,266,267],{"class":95},"?:",[89,269,270],{"class":179}," DiscoverOptions",[89,272,273],{"class":99},")",[89,275,258],{"class":95},[89,277,278],{"class":179}," Promise",[89,280,281],{"class":99},"\u003C",[89,283,284],{"class":179},"DiscoveredUrl",[89,286,287],{"class":99},"[]> {\n",[89,289,291,294,297,299,302],{"class":91,"line":290},19,[89,292,293],{"class":95},"    const",[89,295,296],{"class":99}," target ",[89,298,201],{"class":95},[89,300,301],{"class":179}," resolveDomain",[89,303,304],{"class":99},"(domain, this.name);\n",[89,306,308,310,313,315,318,321],{"class":91,"line":307},20,[89,309,293],{"class":95},[89,311,312],{"class":99}," collector ",[89,314,201],{"class":95},[89,316,317],{"class":95}," new",[89,319,320],{"class":179}," UrlCollector",[89,322,323],{"class":99},"(options, domain);\n",[89,325,327,329,332,334,337,340,343],{"class":91,"line":326},21,[89,328,293],{"class":95},[89,330,331],{"class":99}," api ",[89,333,201],{"class":95},[89,335,336],{"class":159}," `https:\u002F\u002Findex.example\u002Furls?domain=${",[89,338,339],{"class":159},"target",[89,341,342],{"class":159},"}`",[89,344,163],{"class":99},[89,346,348,350,353,355,358,361,364,367,369,371,374,376,378],{"class":91,"line":347},22,[89,349,293],{"class":95},[89,351,352],{"class":99}," rows ",[89,354,201],{"class":95},[89,356,357],{"class":95}," await",[89,359,360],{"class":99}," this.",[89,362,363],{"class":179},"getJSON",[89,365,366],{"class":99},"\u003C{ url",[89,368,258],{"class":95},[89,370,261],{"class":179},[89,372,373],{"class":99},"; seen",[89,375,267],{"class":95},[89,377,261],{"class":179},[89,379,380],{"class":99}," }[]>(api, { signal: options?.signal });\n",[89,382,384],{"class":91,"line":383},23,[89,385,170],{"emptyLinePlaceholder":169},[89,387,389,392,395,398,401,404],{"class":91,"line":388},24,[89,390,391],{"class":95},"    for",[89,393,394],{"class":99}," (",[89,396,397],{"class":95},"const",[89,399,400],{"class":99}," row ",[89,402,403],{"class":95},"of",[89,405,406],{"class":99}," rows) {\n",[89,408,410,413,416,419],{"class":91,"line":409},25,[89,411,412],{"class":95},"      if",[89,414,415],{"class":99}," (collector.done) ",[89,417,418],{"class":95},"break",[89,420,163],{"class":99},[89,422,424,427,430],{"class":91,"line":423},26,[89,425,426],{"class":99},"      collector.",[89,428,429],{"class":179},"push",[89,431,432],{"class":99},"(this.name, row.url, api, row.seen);\n",[89,434,436],{"class":91,"line":435},27,[89,437,438],{"class":99},"    }\n",[89,440,442,444],{"class":91,"line":441},28,[89,443,229],{"class":95},[89,445,446],{"class":99}," collector.results;\n",[89,448,450],{"class":91,"line":449},29,[89,451,238],{"class":99},[89,453,455],{"class":91,"line":454},30,[89,456,457],{"class":99},"}\n",[89,459,461],{"class":91,"line":460},31,[89,462,170],{"emptyLinePlaceholder":169},[89,464,466,469],{"class":91,"line":465},32,[89,467,468],{"class":179},"register",[89,470,471],{"class":99},"(MyIndex, {\n",[89,473,475,478,481],{"class":91,"line":474},33,[89,476,477],{"class":99},"  key: ",[89,479,480],{"class":159},"\"myindex\"",[89,482,483],{"class":99},",\n",[89,485,487],{"class":91,"line":486},34,[89,488,489],{"class":99},"  capabilities: { discover: true },\n",[89,491,493,496,499,502,505,508],{"class":91,"line":492},35,[89,494,495],{"class":179},"  load",[89,497,498],{"class":99},": () ",[89,500,501],{"class":95},"=>",[89,503,504],{"class":99}," Promise.",[89,506,507],{"class":179},"resolve",[89,509,510],{"class":99},"(MyIndex),\n",[89,512,514],{"class":91,"line":513},36,[89,515,516],{"class":99},"});\n",[89,518,520],{"class":91,"line":519},37,[89,521,170],{"emptyLinePlaceholder":169},[89,523,525,527,530,532,534,537,540,542],{"class":91,"line":524},38,[89,526,397],{"class":95},[89,528,529],{"class":99}," mine ",[89,531,201],{"class":95},[89,533,357],{"class":95},[89,535,536],{"class":179}," create",[89,538,539],{"class":99},"(",[89,541,480],{"class":159},[89,543,544],{"class":99},");\n",[546,547,549],"h2",{"id":548},"three-things-not-to-skip","Three things not to skip",[551,552,553,565,578],"ul",{},[554,555,556,564],"li",{},[557,558,559,560,563],"strong",{},"Build the request from ",[86,561,562],{},"resolveDomain()","."," It turns a domain or a full URL into a bare hostname and rejects anything that could sneak a path into your query. Never paste the raw input into a URL.",[554,566,567,573,574,577],{},[557,568,569,570,563],{},"Push through ",[86,571,572],{},"UrlCollector"," It applies every filter, deduplicates, and tells you when the limit is reached through ",[86,575,576],{},"done",". Check it between pages and stop asking.",[554,579,580,586,587,589,590,593,594,597,598,563],{},[557,581,582,583,563],{},"Forward ",[86,584,585],{},"options.signal"," ",[86,588,363],{}," and ",[86,591,592],{},"getTextLines"," on ",[86,595,596],{},"Provider"," take it and add the timeout from ",[86,599,600],{},"create()",[74,602,603,605,606,608,609,616],{},[86,604,592],{}," streams a text answer line by line and cancels the rest once you ",[86,607,418],{},", which is what you want for a CDX dump. ",[610,611,615],"a",{"href":612,"rel":613},"https:\u002F\u002Fgithub.com\u002Fagntn\u002Furls\u002Fblob\u002Fmain\u002Fsrc\u002Fproviders\u002Fwayback.ts",[614],"nofollow","Wayback"," shows the whole pattern.",[546,618,620,621],{"id":619},"after-register","After ",[86,622,623],{},"register()",[74,625,626,627,629,630,633,634,637],{},"The key works with ",[86,628,600],{},", ",[86,631,632],{},"discoverAll()",", the automatic pick and the CLI's ",[86,635,636],{},"-p",", as long as the process that runs them registered it first. Registering the same key again replaces the earlier entry.",[74,639,640,641,644],{},"A source that belongs in the package itself goes into the built-in manifest instead, ",[86,642,643],{},"src\u002Fproviders\u002Findex.ts",", with its own module so it loads lazily. Pull requests welcome.",[646,647,648],"style",{},"html pre.shiki code .skH_V, html code.shiki .skH_V{--shiki-light:var(--shiki-token-keyword);--shiki-default:var(--shiki-token-keyword);--shiki-dark:var(--shiki-token-keyword)}html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":84,"searchDepth":103,"depth":103,"links":650},[651,652],{"id":548,"depth":103,"text":549},{"id":619,"depth":103,"text":653},"After register()","Plug your own passive index into the registry. Subclass Provider and reuse UrlCollector and register it. It shows up on every surface.","md",null,{"icon":658},"i-lucide-plus",{"title":660,"description":654},"Add your own URL source to @agntn\u002Furls","khHEt8f7FmNqdIlIgUGvXgrEDwOWVrfk9thVu87qyt8",[663,665],{"title":24,"path":25,"stem":26,"description":664,"children":-1},"Two tools on MCP and AI SDK and Pi and OMP. A bounded page per source with hasMore and truncated so the agent narrows the call instead of drowning.",{"title":38,"path":34,"stem":35,"description":666,"children":-1},"Web archives and threat intel indexes behind one record shape. Pick a source to see its endpoint and its limits and the traps.",1790685303226]