[{"data":1,"prerenderedAt":606},["ShallowReactive",2],{"navigation_docs":3,"-guide-cli":68,"-guide-cli-surround":601},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":20,"body":70,"description":592,"extension":593,"links":594,"meta":595,"navigation":597,"path":21,"seo":598,"stem":22,"__hash__":600},"docs\u002F1.guide\u002F4.cli.md",{"type":71,"value":72,"toc":580},"minimark",[73,86,114,119,146,165,309,316,321,346,458,464,468,490,497,504,515,518,530,533,537,549,555,559,576],[74,75,76,77,81,82,85],"p",{},"The package ships a ",[78,79,80],"code",{},"urls"," binary. Install it globally or run it through ",[78,83,84],{},"npx",".",[87,88,93],"pre",{"className":89,"code":90,"language":91,"meta":92,"style":92},"language-bash shiki shiki-themes urls urls urls","pnpm add -g @agntn\u002Furls\n","bash","",[78,94,95],{"__ignoreMap":92},[96,97,100,104,108,111],"span",{"class":98,"line":99},"line",1,[96,101,103],{"class":102},"sK71F","pnpm",[96,105,107],{"class":106},"shU9J"," add",[96,109,110],{"class":106}," -g",[96,112,113],{"class":106}," @agntn\u002Furls\n",[115,116,118],"h2",{"id":117},"discover","Discover",[87,120,122],{"className":89,"code":121,"language":91,"meta":92,"style":92},"urls discover example.com -p wayback -n 20\n",[78,123,124],{"__ignoreMap":92},[96,125,126,128,131,134,137,140,143],{"class":98,"line":99},[96,127,80],{"class":102},[96,129,130],{"class":106}," discover",[96,132,133],{"class":106}," example.com",[96,135,136],{"class":106}," -p",[96,138,139],{"class":106}," wayback",[96,141,142],{"class":106}," -n",[96,144,145],{"class":106}," 20\n",[74,147,148,149,152,153,156,157,160,161,164],{},"One URL per line, nothing else, so it pipes straight into ",[78,150,151],{},"grep",", ",[78,154,155],{},"sort -u"," or whatever you feed URLs to. A reader that quits early, like ",[78,158,159],{},"head",", ends the command quietly. No ",[78,162,163],{},"EPIPE"," stack trace confetti.",[166,167,168,181],"table",{},[169,170,171],"thead",{},[172,173,174,178],"tr",{},[175,176,177],"th",{},"Flag",[175,179,180],{},"What it does",[182,183,184,199,209,219,229,239,249,259,269,289,299],"tbody",{},[172,185,186,192],{},[187,188,189],"td",{},[78,190,191],{},"-p, --provider",[187,193,194,195,198],{},"Source key, or ",[78,196,197],{},"all"," for every source",[172,200,201,206],{},[187,202,203],{},[78,204,205],{},"-n, --limit",[187,207,208],{},"Stop after this many URLs per source",[172,210,211,216],{},[187,212,213],{},[78,214,215],{},"-m, --match",[187,217,218],{},"Keep URLs containing one of these substrings",[172,220,221,226],{},[187,222,223],{},[78,224,225],{},"-f, --filter",[187,227,228],{},"Drop URLs containing one of these substrings",[172,230,231,236],{},[187,232,233],{},[78,234,235],{},"-us, --url-scope",[187,237,238],{},"Keep URLs matching a prefix or glob",[172,240,241,246],{},[187,242,243],{},[78,244,245],{},"-uos, --url-out-scope",[187,247,248],{},"Drop URLs matching a prefix or glob",[172,250,251,256],{},[187,252,253],{},[78,254,255],{},"--ext",[187,257,258],{},"Keep these path extensions",[172,260,261,266],{},[187,262,263],{},[78,264,265],{},"--has-query",[187,267,268],{},"Keep URLs with real query keys",[172,270,271,279],{},[187,272,273,152,276],{},[78,274,275],{},"--from",[78,277,278],{},"--to",[187,280,281,282,285,286],{},"Capture date window, ",[78,283,284],{},"2019"," or ",[78,287,288],{},"2019-03-15",[172,290,291,296],{},[187,292,293],{},[78,294,295],{},"-ns, --no-scope",[187,297,298],{},"Turn off the default host check",[172,300,301,306],{},[187,302,303],{},[78,304,305],{},"-j, --jsonl",[187,307,308],{},"One JSON record per line",[74,310,311,312,315],{},"Lists are comma separated. ",[313,314,16],"a",{"href":17}," explains what each filter keeps.",[317,318,320],"h3",{"id":319},"jsonl","JSONL",[87,322,324],{"className":89,"code":323,"language":91,"meta":92,"style":92},"urls discover example.com -p wayback -n 2 -j\n",[78,325,326],{"__ignoreMap":92},[96,327,328,330,332,334,336,338,340,343],{"class":98,"line":99},[96,329,80],{"class":102},[96,331,130],{"class":106},[96,333,133],{"class":106},[96,335,136],{"class":106},[96,337,139],{"class":106},[96,339,142],{"class":106},[96,341,342],{"class":106}," 2",[96,344,345],{"class":106}," -j\n",[87,347,351],{"className":348,"code":349,"language":350,"meta":92,"style":92},"language-json shiki shiki-themes urls urls urls","{\"url\":\"http:\u002F\u002Fexample.com:80\u002F\",\"input\":\"example.com\",\"source\":\"wayback\",\"firstSeen\":\"2002-01-20T14:25:10Z\",\"lastSeen\":\"2002-01-20T14:25:10Z\"}\n{\"url\":\"http:\u002F\u002Fwww.example.com:80\u002F\",\"input\":\"example.com\",\"source\":\"wayback\",\"firstSeen\":\"2002-03-28T01:28:21Z\",\"lastSeen\":\"2002-03-28T01:28:21Z\"}\n","json",[78,352,353,411],{"__ignoreMap":92},[96,354,355,359,362,365,368,371,374,376,379,381,384,386,389,391,394,396,399,401,404,406,408],{"class":98,"line":99},[96,356,358],{"class":357},"s38Sx","{",[96,360,361],{"class":102},"\"url\"",[96,363,364],{"class":357},":",[96,366,367],{"class":106},"\"http:\u002F\u002Fexample.com:80\u002F\"",[96,369,370],{"class":357},",",[96,372,373],{"class":102},"\"input\"",[96,375,364],{"class":357},[96,377,378],{"class":106},"\"example.com\"",[96,380,370],{"class":357},[96,382,383],{"class":102},"\"source\"",[96,385,364],{"class":357},[96,387,388],{"class":106},"\"wayback\"",[96,390,370],{"class":357},[96,392,393],{"class":102},"\"firstSeen\"",[96,395,364],{"class":357},[96,397,398],{"class":106},"\"2002-01-20T14:25:10Z\"",[96,400,370],{"class":357},[96,402,403],{"class":102},"\"lastSeen\"",[96,405,364],{"class":357},[96,407,398],{"class":106},[96,409,410],{"class":357},"}\n",[96,412,414,416,418,420,423,425,427,429,431,433,435,437,439,441,443,445,448,450,452,454,456],{"class":98,"line":413},2,[96,415,358],{"class":357},[96,417,361],{"class":102},[96,419,364],{"class":357},[96,421,422],{"class":106},"\"http:\u002F\u002Fwww.example.com:80\u002F\"",[96,424,370],{"class":357},[96,426,373],{"class":102},[96,428,364],{"class":357},[96,430,378],{"class":106},[96,432,370],{"class":357},[96,434,383],{"class":102},[96,436,364],{"class":357},[96,438,388],{"class":106},[96,440,370],{"class":357},[96,442,393],{"class":102},[96,444,364],{"class":357},[96,446,447],{"class":106},"\"2002-03-28T01:28:21Z\"",[96,449,370],{"class":357},[96,451,403],{"class":102},[96,453,364],{"class":357},[96,455,447],{"class":106},[96,457,410],{"class":357},[74,459,460,461,85],{},"Fields you don't have stay out of the line instead of showing up as ",[78,462,463],{},"null",[317,465,467],{"id":466},"every-source-at-once","Every source at once",[87,469,471],{"className":89,"code":470,"language":91,"meta":92,"style":92},"urls discover example.com -p all -n 20\n",[78,472,473],{"__ignoreMap":92},[96,474,475,477,479,481,483,486,488],{"class":98,"line":99},[96,476,80],{"class":102},[96,478,130],{"class":106},[96,480,133],{"class":106},[96,482,136],{"class":106},[96,484,485],{"class":106}," all",[96,487,142],{"class":106},[96,489,145],{"class":106},[74,491,492,493,496],{},"Each source gets its own block with a count, and a failing source gets one line with its error. With ",[78,494,495],{},"-j"," you get the successful records only, each tagged with its source.",[317,498,500,501],{"id":499},"without-p","Without ",[78,502,503],{},"-p",[74,505,506,507,510,511,514],{},"The CLI picks a source the way ",[78,508,509],{},"discoverWithFallback"," does: configured credentials first, then AlienVault OTX, then the next one when access fails. ",[313,512,12],{"href":513},"\u002Fguide\u002Fdiscover#let-the-library-pick"," has the rules.",[115,516,33],{"id":517},"providers",[87,519,521],{"className":89,"code":520,"language":91,"meta":92,"style":92},"urls providers\n",[78,522,523],{"__ignoreMap":92},[96,524,525,527],{"class":98,"line":99},[96,526,80],{"class":102},[96,528,529],{"class":106}," providers\n",[74,531,532],{},"Prints the registered sources with their endpoints and whether they need a key.",[115,534,536],{"id":535},"mcp-server","MCP server",[87,538,540],{"className":89,"code":539,"language":91,"meta":92,"style":92},"urls mcp\n",[78,541,542],{"__ignoreMap":92},[96,543,544,546],{"class":98,"line":99},[96,545,80],{"class":102},[96,547,548],{"class":106}," mcp\n",[74,550,551,552,554],{},"Starts the MCP server on stdio. ",[313,553,24],{"href":25}," shows the client config.",[115,556,558],{"id":557},"exit-codes","Exit codes",[74,560,561,564,565,568,569,572,573,575],{},[78,562,563],{},"0"," when the command finished, ",[78,566,567],{},"1"," on a bad limit or a failed discovery. A comparison with ",[78,570,571],{},"-p all"," exits ",[78,574,563],{}," even when some sources failed, since their errors are part of the answer.",[577,578,579],"style",{},"html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}",{"title":92,"searchDepth":413,"depth":413,"links":581},[582,589,590,591],{"id":117,"depth":413,"text":118,"children":583},[584,586,587],{"id":319,"depth":585,"text":320},3,{"id":466,"depth":585,"text":467},{"id":499,"depth":585,"text":588},"Without -p",{"id":517,"depth":413,"text":33},{"id":535,"depth":413,"text":536},{"id":557,"depth":413,"text":558},"The urls command. Discover from one source or all of them. Print bare URLs or JSONL. List the sources or start the MCP server.","md",null,{"icon":596},"i-lucide-terminal",true,{"title":599,"description":592},"The urls command line tool for passive URL discovery","xXSbiXC3N9isYwbe4uttP7a1k1g8DVDiSVONJ_IV3I0",[602,604],{"title":16,"path":17,"stem":18,"description":603,"children":-1},"Default host scope. URL prefixes and globs. Extensions and query keys and substrings and dates. What each filter keeps and why the limit counts only survivors.",{"title":24,"path":25,"stem":26,"description":605,"children":-1},"Two tools on MCP and AI SDK and Pi and OMP. A bounded page per source with hasMore and truncated so the agent narrows the call instead of drowning.",1790685303130]