Parser must can process a big count of users (about 1500) in a short time (about 2-3 minutes) with 429 or 403 errors not more than 10-15% of all users. I dont know ho to write parsers and i actually tried write it on python, but its too slow. There is some stupid ideas like, but i dont know how to do it without blocking by steam:/
import axios from 'axios';
import fs from 'fs/promises';
import { SocksProxyAgent } from 'socks-proxy-agent';
async function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async function main() {
// reading configs
const config = JSON.parse(await fs.readFile('config/inventories.json', 'utf-8'));
const players = JSON.parse(await fs.readFile('output/players_output.json', 'utf-8'));
const appid = config.game;
const contextid = config.contextid;
const proxies = config.PROXIES; // SOCKS5
// Reqest parameters
const country = config.country || 'US';
const language = config.language || 'english';
const currency = config.currency || '1'; // USD
const two_factor = config.two_factor || '0';
const bannedProxies = new Set();
for (let i = 0; i < players.length; i++) {
const player = players[i];
const steamid = player.steamid64;
const name = player.name;
//forming url
const baseUrl = `https://steamcommunity.com/inventory/${steamid}/${appid}/${contextid}`;
const url = `${baseUrl}?country=${country}&language=${language}¤cy=${currency}&two_factor=${two_factor}`;
const proxy = proxies[i % proxies.length];
if (bannedProxies.has(proxy)) {
console.log(`Proxy banned, skip: ${proxy}`);
continue;
}
// New socks agent
const agent = new SocksProxyAgent(proxy);
// Browser-like headers
const headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'en-US,en;q=0.9',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
'Referer': baseUrl,
'Origin': 'https://steamcommunity.com',
'Host': 'steamcommunity.com',
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'Cache-Control': 'no-cache',
};
try {
console.log(`Steam Inventory reqesest for ${name} with proxy: ${proxy}`);
const response = await axios.get(url, {
httpsAgent: agent,
headers,
timeout: 10000,
});
console.log(`Status: ${response.status}`);
if (response.status === 429 || response.status === 403) {
console.log(`Proxy has been banned: ${proxy}`);
bannedProxies.add(proxy);
continue;
}
// Working with response data
} catch (err) {
console.log(`Proxy error ${proxy}: ${err.message}`);
bannedProxies.add(proxy);
continue;
}
// Pause
await sleep(500);
}
}
main().catch(console.error);
And my JSON configs like after nuclear war (There is a huge count of trash)
{
"steam_api_key": "KEY",
"database": {
"filename": "data/data.db"
},
"PLAYERS": "output/players_output.json",
"URL": "https://steamcommunity/inventory",
"DELAY": 0.5,
"TIMEOUT": 1.5,
"game": 730,
"contextid": 2,
"mode": "scrape",
"searchFor": null,
"outputFile": "./output/inventories.json",
"USE_PROXY": true,
"PROXY_CHANGERATE": 2,
"PROXIES": [
there is 10 proxies
]
}
Output must be like this in future (its not full code, because im stuck on steam blocking)
[
{
"name": "",
"steamid64": "",
"address": ""
}, ...
]
And input looks like
[
{
"nickname": "",
"steamid": "",
"server": "its frome another script, nothing to do there, only for next",
"inventory": [
{
"id": "45072097998",
"market_hash_name": "Dreams & Nightmares Case",
"type": "Container, base class"
}, ...
]
}, ...
]
