User-agent: * Allow: / # Disallow LivePerson Chat Test application Disallow: /LivePersonTest* # Disallow General CMS structure not needed for crawlers Disallow: /defaultMultiSites.aspx Disallow: /ScriptResource.axd Disallow: /WebResource.axd Disallow: /docs/ Disallow: /publish/ Disallow: /ClickTaleCache.ashx Disallow: /apps/ Disallow: /ProductDetailHandler Disallow: /webshop/ Disallow: /pro_1.asp Disallow: /pro_detail.asp #Disallow News entry of microsite "HLM" Disallow: /newscontentgeneric.WIKA?AxID=462 Disallow: /newscontentgeneric.WIKA?AxID=467 Disallow: /newscontentgeneric.WIKA?AxID=469 Disallow: /newscontentgeneric.WIKA?AxID=473 Disallow: /newscontentgeneric.WIKA?AxID=474 Disallow: /newscontentgeneric.WIKA?AxID=475 Disallow: /newscontentgeneric.WIKA?AxID=470 Disallow: /newscontentgeneric.WIKA?AxID=471 Disallow: /newscontentgeneric.WIKA?AxID=472 Disallow: /newscontentgeneric.WIKA?AxID=468 Disallow: /newscontentgeneric.WIKA?AxID=452 Disallow: /newscontentgeneric.WIKA?AxID=457 Disallow: /newscontentgeneric.WIKA?AxID=458 Disallow: /newscontentgeneric.WIKA?AxID=459 Disallow: /newscontentgeneric.WIKA?AxID=460 Disallow: /newscontentgeneric.WIKA?AxID=461 Disallow: /newscontentgeneric.WIKA?AxID=462 Disallow: /newscontentgeneric.WIKA?AxID=463 Disallow: /newscontentgeneric.WIKA?AxID=466 Disallow: /newscontentgeneric.WIKA?AxID=453 Disallow: /newscontentgeneric.WIKA?AxID=454 # Disallow indexed news entry for en-co Disallow: http://en-co.wika.de/newscontentgeneric.WIKA?AxID=4250 Disallow: http://en-co.wika.de/newscontentgeneric.WIKA?AxID=4259 # Disallow indexed news entry of KSR Kuebler for WIKA.ES Disallow: http://www.wika.es/newscontentgeneric.WIKA?AxID=2870 Disallow: http://www.wika.es/newscontentgeneric.WIKA?AxID=2920 Disallow: http://www.wika.es/newscontentgeneric.WIKA?AxID=2839 Disallow: www.wika.es/newscontentgeneric.WIKA?AxID=2870 Disallow: www.wika.es/newscontentgeneric.WIKA?AxID=2920 Disallow: www.wika.es/newscontentgeneric.WIKA?AxID=2839 # Disallow indexed news entry for WIKA.FR Disallow: http://www.wika.fr/newscontentgeneric.WIKA?AxID=2819 Disallow: http://www.wika.fr/newscontentgeneric.WIKA?AxID=3011 # Disallow indexed news entry for WIKA.RU Disallow: http://www.wika.ru/newscontentgeneric.WIKA?AxID=240 # Disallow Sensymic brochures for all Domains Disallow: /upload/BR_SensyMICProductCatalogue_de_ds_100882.pdf Disallow: /upload/BR_SensyMICProductCatalogue_en_ds_100892.pdf # Disallow PG Datasheets Disallow: /templates/pdfs/DS_PG23ST_de_de.pdf Disallow: /templates/pdfs/DS_PG23ST_en_co.pdf # Disallow special crawler making us problems User-agent: WebCopier Disallow: / User-agent: WebCopier v4.0 Disallow: / User-agent: WebCopier+v4.0 Disallow: / User-agent: Microsoft+Office+Protocol+Discovery Disallow: / User-agent: infometrics-bot, http://www.infometrics.de Disallow: / # Disallow other crawlers made us CPU load User-agent: 360Spider Disallow: / User-agent: 8LEGS Disallow: / User-agent: AhrefsBot Disallow: / User-agent: betaBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: BoardReader Disallow: / User-agent: BPImageWalker Disallow: / User-agent: CheckMarkNetwork Disallow: / User-agent: Clockwork Data Vault Disallow: / User-agent: coccoc Disallow: / User-agent: CRAZYWEBCRAWLER Disallow: / User-agent: Domain Re-Animator Disallow: / User-agent: DomainStatsBot Disallow: / User-agent: DotBot Disallow: / User-agent: dotbot Disallow: / User-agent: EasouSpider Disallow: / # Ezooms and dotbot User-agent: Ezooms Disallow: / User-agent: Findxbot Disallow: / User-agent: GetIntent Disallow: / User-agent: Grammarly Disallow: / User-agent: Guardcrwlr Disallow: / User-agent: HubSpot Links Crawler Disallow: / User-agent: istellabot Disallow: / User-agent: KOCMOHABT Disallow: / #User-agent: link checker #Disallow: / #User-agent: linkcheck #Disallow: / User-agent: LinksCrawler Disallow: / User-agent: linkdexbot Disallow: / #User-agent: Link Sleuth #Disallow: / User-agent: LinkSpammer Disallow: / User-agent: LinkWalker Disallow: / User-agent: ltx71 Disallow: / User-agent: Mail.RU_Bot Disallow: / User-Agent: MJ12bot Disallow: / User-Agent: NetTrack Disallow: / User-agent: okhttp Disallow: / User-agent: OpenLinkProfiler Disallow: / User-Agent: PaperLiBot Disallow: / User-agent: Pingdom Disallow: / User-agent: proximic Disallow: / User-agent: Pu_iN Disallow: / User-agent: Pulsepoint Disallow: / User-agent: Qwantify Disallow: / User-agent: RankvalBot Disallow: / User-agent: SalesIntelligent Disallow: / User-agent: Scrapy Disallow: / User-agent: Searchie Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: SocialRankIOBot Disallow: / User-agent: Sogou Disallow: / User-agent: Sogou web spider Disallow: / User-agent: spbot Disallow: / User-agent: SurveyBot Disallow: / User-agent: Uptimebot Disallow: / User-agent: Vegi bot Disallow: / User-agent: VeriCiteCrawler Disallow: / User-agent: webmeup-crawler Disallow: / User-agent: weborama-fetcher Disallow: / User-agent: YisouSpider Disallow: / User-agent: YoudaoBot Disallow: / # Reference to sitemap XML Sitemap: http://en.wika.com/sitemapindex.xml