# Robots.txt for ATAKNAS BANTEN - Asosiasi Tenaga Ahli Konstruksi Nasional Provinsi Banten # Lembaga Sertifikasi Profesi (LSP) Terakreditasi BNSP # Allow all legitimate search engines User-agent: * Allow: / # Block bad bots and scrapers User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: YandexBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: facebookexternalhit Disallow: / User-agent: Twitterbot Disallow: / User-agent: LinkedInBot Disallow: / User-agent: WhatsApp Disallow: / User-agent: TelegramBot Disallow: / User-agent: SlackBot Disallow: / User-agent: Applebot Disallow: / User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # Block common bad bots User-agent: 008 Disallow: / User-agent: 360Spider Disallow: / User-agent: Aboundexbot Disallow: / User-agent: AcoonBot Disallow: / User-agent: AddThis.com Disallow: / User-agent: ADmantX Disallow: / User-agent: AhrefsBot Disallow: / User-agent: AlexaBot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BUbiNG Disallow: / User-agent: CCBot Disallow: / User-agent: Charlotte Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Coccoc Disallow: / User-agent: Crawler Disallow: / User-agent: Crawlera Disallow: / User-agent: Curious Disallow: / User-agent: Custo Disallow: / User-agent: Default Browser 0 Disallow: / User-agent: DIIbot Disallow: / User-agent: Domain Re-Animator Bot Disallow: / User-agent: DotBot Disallow: / User-agent: DuckDuckBot Disallow: / User-agent: EasouSpider Disallow: / User-agent: ExaBot Disallow: / User-agent: FacebookBot Disallow: / User-agent: FeedFetcher-Google Disallow: / User-agent: Feedly Disallow: / User-agent: Fever Disallow: / User-agent: Flipboard Disallow: / User-agent: Generic Disallow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / User-agent: GrapeshotCrawler Disallow: / User-agent: HeadlessChrome Disallow: / User-agent: Heritrix Disallow: / User-agent: HTTP_Request2 Disallow: / User-agent: HubSpot Disallow: / User-agent: ia_archiver Disallow: / User-agent: ICC-Crawler Disallow: / User-agent: ichiro Disallow: / User-agent: IIS Application Disallow: / User-agent: IIS_Application Disallow: / User-agent: InfoWizards Disallow: / User-agent: IRLbot Disallow: / User-agent: KaloogaBot Disallow: / User-agent: KikBot Disallow: / User-agent: Kouio Disallow: / User-agent: Larbin Disallow: / User-agent: Linkdexbot Disallow: / User-agent: LinkedInBot Disallow: / User-agent: LinkpadBot Disallow: / User-agent: LivelapBot Disallow: / User-agent: LoadImpactPageAnalyzer Disallow: / User-agent: LoadImpactRload Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: Mappydata Disallow: / User-agent: Mediapartners-Google Allow: / User-agent: MJ12bot Disallow: / User-agent: MojeekBot Disallow: / User-agent: NalezenCzBot Disallow: / User-agent: NerdyBot Disallow: / User-agent: NetcraftSurveyAgent Disallow: / User-agent: Nimbostratus-Bot Disallow: / User-agent: Nutch Disallow: / User-agent: oBot Disallow: / User-agent: Ocarinabot Disallow: / User-agent: Ocelli Disallow: / User-agent: OkHttp Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: OnetBot Disallow: / User-agent: OrangeBot Disallow: / User-agent: Owler Disallow: / User-agent: PagePeeker Disallow: / User-agent: PaperLiBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Pinterestbot Disallow: / User-agent: Pizilla Disallow: / User-agent: Plukkie Disallow: / User-agent: Proximic Disallow: / User-agent: QuerySeekerSpider Disallow: / User-agent: RedditBot Disallow: / User-agent: Riddler Disallow: / User-agent: Rogerbot Disallow: / User-agent: SafeDNSBot Disallow: / User-agent: ScoutJet Disallow: / User-agent: Scrapy Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: SemrushBot Disallow: / User-agent: SeoCheckBot Disallow: / User-agent: SeoCheckBot Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: SEOkicks Disallow: / User-agent: Seoscanners Disallow: / User-agent: Seoscanners.net Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: Siteimprove Disallow: / User-agent: SiteLockSpider Disallow: / User-agent: Sogou Disallow: / User-agent: Sogou web spider Disallow: / User-agent: Sosospider Disallow: / User-agent: spbot Disallow: / User-agent: SputnikBot Disallow: / User-agent: SurveyBot Disallow: / User-agent: TagoBot Disallow: / User-agent: TelegramBot Disallow: / User-agent: TestBot Disallow: / User-agent: The Knowledge AI Disallow: / User-agent: The Knowledge AI Disallow: / User-agent: TweetmemeBot Disallow: / User-agent: Twitterbot Disallow: / User-agent: Uptimebot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: Vercelbot Disallow: / User-agent: VindigoBot Disallow: / User-agent: VoilaBot Disallow: / User-agent: Voltron Disallow: / User-agent: W3C_Validator Disallow: / User-agent: WebSitePulse Disallow: / User-agent: WeSEE:Search Disallow: / User-agent: WeSEE Disallow: / User-agent: Wotbox Disallow: / User-agent: Wotbox Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexImages Disallow: / User-agent: YandexMetrika Disallow: / User-agent: YandexMobileBot Disallow: / User-agent: YandexNews Disallow: / User-agent: YandexPagechecker Disallow: / User-agent: YandexSearchShop Disallow: / User-agent: YandexVertis Disallow: / User-agent: YandexVertis Disallow: / User-agent: YisouSpider Disallow: / User-agent: Zao Disallow: / User-agent: Zao Disallow: / User-agent: Zookabot Disallow: / User-agent: ZumBot Disallow: / # Disallow access to sensitive areas Disallow: /app/ Disallow: /lib/ Disallow: /vendors/ Disallow: /tmp/ Disallow: /Config/ Disallow: /Console/ Disallow: /Model/ Disallow: /Plugin/ Disallow: /Test/ Disallow: /nbproject/ Disallow: /plugins/ Disallow: /vendor/ Disallow: /webroot/files/ Disallow: /webroot/documentation/ Disallow: /webroot/snippets/ Disallow: /webroot/vendor/ Disallow: /webroot/restore.php Disallow: /webroot/test.php Disallow: /*.log$ Disallow: /*.sql$ Disallow: /*.txt$ Disallow: /*.xml$ Disallow: /*.json$ Disallow: /*.php$ Disallow: /*?* Disallow: /*&* # Allow important pages Allow: /about Allow: /skk-konstruksi Allow: /sbu Allow: /blog Allow: /contact Allow: /sitemap.xml # Sitemap location Sitemap: https://ataknasbanten.com/sitemap.xml # Crawl delay for good bots User-agent: Googlebot Crawl-delay: 1 User-agent: Bingbot Crawl-delay: 1 User-agent: * Crawl-delay: 2