Skip to content

Commit 6d749b4

Browse files
committed
feat: add geo_fanout tool with first-class classified geo results
geo_fanout fetches the same URL from multiple country exits in parallel (Web Unlocker country targeting) and returns one structured report. A geo that is blocked (403/451), redirected (3xx to a different host), rate-limited (429) or fails transiently becomes a FIRST-CLASS classified result rather than a discarded error, so geo-gating and regional access/price differences are observable at a glance. geo_utils.js holds the pure, table-tested aggregation logic (normalize_geos, build_geo_entry, summarize_fanout) built on the brightdata#104 classify_response taxonomy. The tool is registered in default/pro mode and in the advanced_scraping group.
1 parent f7c2e34 commit 6d749b4

6 files changed

Lines changed: 453 additions & 1 deletion

File tree

‎geo_utils.js‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
'use strict'; /*jslint node:true es9:true*/
2+
3+
// Pure helpers for the geo_fanout tool: validating the list of country exits
4+
// and turning a set of per-geo results into a single structured report where a
5+
// blocked / redirected / failed geo is a FIRST-CLASS classified result, not a
6+
// discarded error. No network or transport dependencies live here so the fanout
7+
// aggregation logic is unit-testable in isolation.
8+
9+
import {classify_response, OUTCOME} from './retry_utils.js';
10+
11+
// Per-geo status the caller sees in the aggregated report.
12+
export const GEO_STATUS = {
13+
OK: 'ok', // fetched successfully
14+
BLOCKED: 'blocked', // target refused this geo (403/451)
15+
REDIRECTED: 'redirected', // 3xx to a different host/path (geo gating signal)
16+
RATE_LIMITED: 'rate_limited',
17+
ERROR: 'error', // transient/fatal failure after retries
18+
};
19+
20+
// Validate and normalize a list of 2-letter ISO country codes. Lowercases,
21+
// trims, dedupes (preserving first-seen order), and rejects malformed entries
22+
// loudly so a typo never becomes a silently-dropped exit. Returns the clean
23+
// array; throws Error on any invalid code (callers want fast, explicit failure).
24+
export function normalize_geos(geos){
25+
if (!Array.isArray(geos) || geos.length===0)
26+
throw new Error('geos must be a non-empty array of 2-letter codes');
27+
const seen = new Set();
28+
const out = [];
29+
for (const raw of geos)
30+
{
31+
if (typeof raw!='string')
32+
throw new Error(`invalid geo (not a string): ${JSON.stringify(raw)}`);
33+
const code = raw.trim().toLowerCase();
34+
if (!/^[a-z]{2}$/.test(code))
35+
throw new Error(`invalid geo country code: "${raw}" `
36+
+`(expected 2 letters, e.g. "us", "de")`);
37+
if (seen.has(code))
38+
continue;
39+
seen.add(code);
40+
out.push(code);
41+
}
42+
return out;
43+
}
44+
45+
// Detect whether a successful-looking response is actually a geo redirect.
46+
// A 3xx with a Location header pointing at a different host is the classic
47+
// "Belgian visitor bounced to a different storefront" signal we must capture.
48+
function detect_redirect(status, headers, request_url){
49+
if (!(status>=300 && status<400))
50+
return null;
51+
const location = read_header(headers, 'location');
52+
if (!location)
53+
return {redirected: true, location: null, cross_host: false};
54+
let cross_host = false;
55+
try {
56+
const from = new URL(request_url);
57+
const to = new URL(location, request_url);
58+
cross_host = from.host!==to.host;
59+
} catch(e){
60+
cross_host = false;
61+
}
62+
return {redirected: true, location, cross_host};
63+
}
64+
65+
function read_header(headers, name){
66+
if (!headers || typeof headers!='object')
67+
return undefined;
68+
const target = name.toLowerCase();
69+
for (const key of Object.keys(headers))
70+
{
71+
if (key.toLowerCase()===target)
72+
return headers[key];
73+
}
74+
return undefined;
75+
}
76+
77+
// Map one classified outcome to the per-geo report status.
78+
function outcome_to_geo_status(outcome){
79+
switch (outcome)
80+
{
81+
case OUTCOME.SUCCESS: return GEO_STATUS.OK;
82+
case OUTCOME.BLOCKED: return GEO_STATUS.BLOCKED;
83+
case OUTCOME.RATE_LIMITED: return GEO_STATUS.RATE_LIMITED;
84+
default: return GEO_STATUS.ERROR;
85+
}
86+
}
87+
88+
// Build the per-geo entry for a single settled attempt. `attempt` is the
89+
// normalized shape produced by the tool's executor:
90+
// {geo, url, response:{status, headers, exit_ip?}} on a completed request, or
91+
// {geo, url, error:{code?, response?}} on a thrown failure.
92+
// `now_ms` is injectable for deterministic Retry-After math under test.
93+
export function build_geo_entry(attempt, now_ms = Date.now()){
94+
const obj = attempt && typeof attempt=='object' ? attempt : {};
95+
const geo = typeof obj.geo=='string' ? obj.geo : 'unknown';
96+
const url = typeof obj.url=='string' ? obj.url : null;
97+
// Normalize to the shape classify_response understands: a thrown error keeps
98+
// its {error} envelope; a completed request passes its {status, headers}.
99+
const classify_input = obj.error ? {error: obj.error}
100+
: (obj.response || {});
101+
const classification = classify_response(classify_input, now_ms);
102+
const status = classification.status;
103+
const headers = obj.response ? obj.response.headers
104+
: (obj.error && obj.error.response
105+
? obj.error.response.headers : undefined);
106+
const exit_ip = obj.response
107+
&& typeof obj.response.exit_ip=='string'
108+
? obj.response.exit_ip : null;
109+
110+
const redirect = detect_redirect(status, headers, url);
111+
let geo_status = outcome_to_geo_status(classification.outcome);
112+
if (redirect && redirect.redirected)
113+
geo_status = GEO_STATUS.REDIRECTED;
114+
115+
return {
116+
geo,
117+
url,
118+
status: geo_status,
119+
http_status: status,
120+
exit_ip,
121+
outcome: classification.outcome,
122+
retry_after_ms: classification.retry_after_ms,
123+
reason: classification.reason,
124+
redirect: redirect || null,
125+
};
126+
}
127+
128+
// Aggregate all per-geo entries into one report. Every geo appears exactly once
129+
// regardless of success/failure; nothing is dropped. The summary makes the
130+
// "this geo was blocked / redirected" fact queryable at a glance.
131+
export function summarize_fanout(entries){
132+
const list = Array.isArray(entries) ? entries : [];
133+
const summary = {
134+
total: list.length,
135+
ok: 0,
136+
blocked: 0,
137+
redirected: 0,
138+
rate_limited: 0,
139+
error: 0,
140+
};
141+
for (const e of list)
142+
{
143+
switch (e.status)
144+
{
145+
case GEO_STATUS.OK: summary.ok++; break;
146+
case GEO_STATUS.BLOCKED: summary.blocked++; break;
147+
case GEO_STATUS.REDIRECTED: summary.redirected++; break;
148+
case GEO_STATUS.RATE_LIMITED: summary.rate_limited++; break;
149+
default: summary.error++; break;
150+
}
151+
}
152+
return {
153+
summary,
154+
any_blocked: summary.blocked>0,
155+
any_redirected: summary.redirected>0,
156+
results: list,
157+
};
158+
}

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
"server.js",
4141
"search_utils.js",
4242
"retry_utils.js",
43+
"geo_utils.js",
4344
"browser_tools.js",
4445
"browser_session.js",
4546
"aria_snapshot_filter.js",

‎server.js‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import prompts from './prompts.js';
88
import {GROUPS} from './tool_groups.js';
99
import {parse_google_search_response} from './search_utils.js';
1010
import {classify_response, should_retry} from './retry_utils.js';
11+
import {normalize_geos, build_geo_entry, summarize_fanout}
12+
from './geo_utils.js';
1113
import {createRequire} from 'node:module';
1214
import {remark} from 'remark';
1315
import strip from 'strip-markdown';
@@ -23,7 +25,7 @@ const base_timeout = process.env.BASE_TIMEOUT
2325
const base_max_retries = Math.min(
2426
parseInt(process.env.BASE_MAX_RETRIES || '0', 10), 3);
2527
const pro_mode_tools = ['search_engine', 'scrape_as_markdown',
26-
'search_engine_batch', 'scrape_batch', 'discover'];
28+
'search_engine_batch', 'scrape_batch', 'discover', 'geo_fanout'];
2729
const tool_groups = process.env.GROUPS ?
2830
process.env.GROUPS.split(',').map(g=>g.trim().toLowerCase())
2931
.filter(Boolean) : [];
@@ -411,6 +413,69 @@ addTool({
411413
}),
412414
});
413415

416+
addTool({
417+
name: 'geo_fanout',
418+
description: 'Fetch the SAME url from multiple country exits in parallel and '
419+
+'return one structured report. A geo that is blocked (403/451), '
420+
+'redirected (3xx to a different host), rate-limited (429) or fails '
421+
+'transiently becomes a FIRST-CLASS classified result - not a discarded '
422+
+'error. Ideal for detecting geo-gating, regional price/availability '
423+
+'differences, and access denial across countries.',
424+
annotations: {
425+
title: 'Geo Fanout',
426+
readOnlyHint: true,
427+
openWorldHint: true,
428+
},
429+
parameters: z.object({
430+
url: z.string().url(),
431+
countries: z.array(z.string().length(2))
432+
.min(1)
433+
.max(10)
434+
.describe('2-letter ISO country codes to fan the request across '
435+
+'(e.g., ["de", "be", "fr"]). Deduped; max 10.'),
436+
data_format: z.enum(['raw', 'markdown'])
437+
.optional()
438+
.default('markdown')
439+
.describe('Response body format per geo (default: markdown).'),
440+
}),
441+
execute: tool_fn('geo_fanout', async({url, countries, data_format}, ctx)=>{
442+
const geos = normalize_geos(countries);
443+
const now = Date.now();
444+
const attempts = geos.map(geo=>(async()=>{
445+
try {
446+
const response = await base_request({
447+
url: 'https://api.brightdata.com/request',
448+
method: 'POST',
449+
data: {
450+
url,
451+
zone: unlocker_zone,
452+
format: 'raw',
453+
data_format,
454+
country: geo,
455+
},
456+
headers: api_headers(ctx.clientName, 'geo_fanout'),
457+
responseType: 'text',
458+
});
459+
const exit_ip = response.headers?.['x-brd-ip']
460+
|| response.headers?.['x-luminati-ip'] || null;
461+
return build_geo_entry({
462+
geo,
463+
url,
464+
response: {
465+
status: response.status,
466+
headers: response.headers,
467+
...exit_ip ? {exit_ip} : {},
468+
},
469+
}, now);
470+
} catch(e){
471+
return build_geo_entry({geo, url, error: e}, now);
472+
}
473+
})());
474+
const entries = await Promise.all(attempts);
475+
return JSON.stringify(summarize_fanout(entries), null, 2);
476+
}),
477+
});
478+
414479
addTool({
415480
name: 'scrape_as_html',
416481
description: 'Scrape a single webpage URL with advanced options for '

‎test/geo-fanout-tool.test.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
'use strict'; /*jslint node:true es9:true*/
2+
import test from 'node:test';
3+
import assert from 'node:assert/strict';
4+
import {fileURLToPath} from 'node:url';
5+
import {dirname, resolve} from 'node:path';
6+
import {Client} from '@modelcontextprotocol/sdk/client/index.js';
7+
import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
8+
9+
const test_dir = dirname(fileURLToPath(import.meta.url));
10+
const repo_root = resolve(test_dir, '..');
11+
12+
test('geo_fanout tool is registered and well-formed over stdio', async()=>{
13+
const env = {
14+
...process.env,
15+
API_TOKEN: 'dummy-token',
16+
PRO_MODE: 'true',
17+
};
18+
const client = new Client(
19+
{name: 'geo-fanout-test', version: '0.0.1'},
20+
{capabilities: {tools: {}}});
21+
const transport = new StdioClientTransport({
22+
command: process.execPath,
23+
args: ['server.js'],
24+
cwd: repo_root,
25+
env,
26+
});
27+
try {
28+
await client.connect(transport);
29+
const {tools} = await client.listTools();
30+
const geo_fanout = tools.find(tool=>tool.name=='geo_fanout');
31+
assert.ok(geo_fanout, 'geo_fanout tool is exposed');
32+
assert.match(geo_fanout.description, /first-class/i,
33+
'description documents first-class classified results');
34+
const props = geo_fanout.inputSchema?.properties || {};
35+
assert.ok(props.url, 'geo_fanout exposes a url parameter');
36+
assert.ok(props.countries, 'geo_fanout exposes a countries parameter');
37+
} finally {
38+
await client.close();
39+
}
40+
});

0 commit comments

Comments
 (0)