#robots.txt

User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-CT
User-agent: SiteAuditBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
User-agent: Barkrowler
User-agent: SEOkicks
User-agent: SEOkicks-Robot
User-agent: linkdexbot
User-agent: SeobilityBot
User-agent: ZoomBot
User-agent: Cocolyzebot
User-agent: SiteCheckerBotCrawler
User-agent: DomainStatsBot
User-agent: SenutoBot
User-agent: online-webceo-bot
User-agent: SEOlizer
User-agent: hypestat
User-agent: BrightEdge Crawler
User-agent: Screaming Frog SEO Spider
User-agent: sitebulb
User-agent: Sogou web spider
User-agent: Exabot
User-agent: Gigabot
Disallow: /


User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: Webzio-Extended
User-agent: PanguBot
User-agent: AI2Bot
User-agent: Kangaroo Bot
User-agent: DuckAssistBot
User-agent: img2dataset
Disallow: /


User-agent: HTTrack
User-agent: WebCopier
User-agent: WebZIP
User-agent: Wget
User-agent: curl
User-agent: python-requests
User-agent: Scrapy
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: ExtractorPro
User-agent: WebStripper
User-agent: SiteSnagger
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: Zealbot
User-agent: grub-client
Disallow: /


User-agent: CensysInspect
User-agent: Expanse
User-agent: InternetMeasurement
User-agent: NetcraftSurveyAgent
User-agent: zgrab
User-agent: masscan
Disallow: /



User-agent: Googlebot-Image
Disallow: /2.0/big-logos/

User-agent: coccoc
Disallow: /

User-agent: Crawlera
Disallow: /

User-agent: Kyoto-Tohoku-Crawler
Disallow: /

User-agent: Kyoto-Crawler
Disallow: /

User-agent: Sogou Spider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: *
Disallow: /search.html
Disallow: /apisearch.html
Disallow: /gotojob.html
Disallow: /my_jobs.html
Disallow: /my_alerts.html
Disallow: /my_account.html
Disallow: /login.html
Disallow: /my_search.html
Disallow: /directjob.html
Disallow: /url.html
Disallow: /sponsorjob.html
Disallow: /gotoad.html
Disallow: /GetJobAt/http/
Disallow: /recruitnet/get_new_job_count.jsp
Disallow: /linkedinlogin_.html
Disallow: /facebooklogin_.html
Disallow: /linkedinlogin.html
Disallow: /advanced_search.html
Disallow: /trackerror.html
Disallow: /demo/*
Disallow: /recruitnet/sg/*
Disallow: /recruit/*


Disallow: /recruitnet/all/googleonetap.html

Sitemap: https://www.recruit.net/sitemaps/jobs/jobs_index.xml