pastebin

Paste Search Dynamic
Recent pastes
Facebook.com/robots.txt
  1. # Notice: Collection of data on Facebook through automated means is
  2. # prohibited unless you have express written permission from Facebook
  3. # and may only be conducted for the limited purpose contained in said
  4. # permission.
  5. # See: http://www.facebook.com/apps/site_scraping_tos_terms.php
  6.  
  7. user-agent: Applebot
  8. disallow: /ajax/
  9. disallow: /album.php
  10. disallow: /checkpoint/
  11. disallow: /contact_importer/
  12. disallow: /dialog/
  13. disallow: /fbml/ajax/dialog/
  14. disallow: /feeds/
  15. disallow: /file_download.php
  16. disallow: /hashtag/
  17. disallow: /l.php
  18. disallow: /live/
  19. disallow: /moments_app/
  20. disallow: /p.php
  21. disallow: /photo.php
  22. disallow: /photos.php
  23. disallow: /share.php
  24. disallow: /share/
  25. disallow: /sharer.php
  26. disallow: /sharer/
  27.  
  28. user-agent: baiduspider
  29. disallow: /ajax/
  30. disallow: /album.php
  31. disallow: /checkpoint/
  32. disallow: /contact_importer/
  33. disallow: /dialog/
  34. disallow: /fbml/ajax/dialog/
  35. disallow: /feeds/
  36. disallow: /file_download.php
  37. disallow: /hashtag/
  38. disallow: /l.php
  39. disallow: /live/
  40. disallow: /moments_app/
  41. disallow: /p.php
  42. disallow: /photo.php
  43. disallow: /photos.php
  44. disallow: /share.php
  45. disallow: /share/
  46. disallow: /sharer.php
  47. disallow: /sharer/
  48.  
  49. user-agent: Bingbot
  50. disallow: /ajax/
  51. disallow: /album.php
  52. disallow: /checkpoint/
  53. disallow: /contact_importer/
  54. disallow: /dialog/
  55. disallow: /fbml/ajax/dialog/
  56. disallow: /feeds/
  57. disallow: /file_download.php
  58. disallow: /hashtag/
  59. disallow: /l.php
  60. disallow: /live/
  61. disallow: /moments_app/
  62. disallow: /p.php
  63. disallow: /photo.php
  64. disallow: /photos.php
  65. disallow: /share.php
  66. disallow: /share/
  67. disallow: /sharer.php
  68. disallow: /sharer/
  69.  
  70. user-agent: Discordbot
  71. disallow: /ajax/
  72. disallow: /album.php
  73. disallow: /checkpoint/
  74. disallow: /contact_importer/
  75. disallow: /dialog/
  76. disallow: /fbml/ajax/dialog/
  77. disallow: /feeds/
  78. disallow: /file_download.php
  79. disallow: /hashtag/
  80. disallow: /l.php
  81. disallow: /live/
  82. disallow: /moments_app/
  83. disallow: /p.php
  84. disallow: /photo.php
  85. disallow: /photos.php
  86. disallow: /share.php
  87. disallow: /share/
  88. disallow: /sharer.php
  89. disallow: /sharer/
  90.  
  91. user-agent: facebookexternalhit
  92. disallow: /ajax/
  93. disallow: /album.php
  94. disallow: /checkpoint/
  95. disallow: /contact_importer/
  96. disallow: /dialog/
  97. disallow: /fbml/ajax/dialog/
  98. disallow: /feeds/
  99. disallow: /file_download.php
  100. disallow: /hashtag/
  101. disallow: /l.php
  102. disallow: /live/
  103. disallow: /moments_app/
  104. disallow: /p.php
  105. disallow: /photo.php
  106. disallow: /photos.php
  107. disallow: /share.php
  108. disallow: /share/
  109. disallow: /sharer.php
  110. disallow: /sharer/
  111.  
  112. user-agent: Googlebot
  113. disallow: /ajax/
  114. disallow: /album.php
  115. disallow: /checkpoint/
  116. disallow: /contact_importer/
  117. disallow: /dialog/
  118. disallow: /fbml/ajax/dialog/
  119. disallow: /feeds/
  120. disallow: /file_download.php
  121. disallow: /hashtag/
  122. disallow: /l.php
  123. disallow: /live/
  124. disallow: /moments_app/
  125. disallow: /p.php
  126. disallow: /photo.php
  127. disallow: /photos.php
  128. disallow: /share.php
  129. disallow: /share/
  130. disallow: /sharer.php
  131. disallow: /sharer/
  132.  
  133. user-agent: Googlebot-Image
  134. disallow: /ajax/
  135. disallow: /album.php
  136. disallow: /checkpoint/
  137. disallow: /contact_importer/
  138. disallow: /dialog/
  139. disallow: /fbml/ajax/dialog/
  140. disallow: /feeds/
  141. disallow: /file_download.php
  142. disallow: /hashtag/
  143. disallow: /l.php
  144. disallow: /live/
  145. disallow: /map_tile.php
  146. disallow: /moments_app/
  147. disallow: /p.php
  148. disallow: /photo.php
  149. disallow: /photos.php
  150. disallow: /share.php
  151. disallow: /share/
  152. disallow: /sharer.php
  153. disallow: /sharer/
  154. disallow: /static_map.php
  155.  
  156. user-agent: ia_archiver
  157. disallow: /ajax/
  158. disallow: /album.php
  159. disallow: /checkpoint/
  160. disallow: /contact_importer/
  161. disallow: /dialog/
  162. disallow: /fbml/ajax/dialog/
  163. disallow: /feeds/
  164. disallow: /file_download.php
  165. disallow: /hashtag/
  166. disallow: /l.php
  167. disallow: /live/
  168. disallow: /moments_app/
  169. disallow: /p.php
  170. disallow: /photo.php
  171. disallow: /photos.php
  172. disallow: /share.php
  173. disallow: /share/
  174. disallow: /sharer.php
  175. disallow: /sharer/
  176.  
  177. user-agent: LinkedInBot
  178. disallow: /ajax/
  179. disallow: /album.php
  180. disallow: /checkpoint/
  181. disallow: /contact_importer/
  182. disallow: /dialog/
  183. disallow: /fbml/ajax/dialog/
  184. disallow: /feeds/
  185. disallow: /file_download.php
  186. disallow: /hashtag/
  187. disallow: /l.php
  188. disallow: /live/
  189. disallow: /moments_app/
  190. disallow: /p.php
  191. disallow: /photo.php
  192. disallow: /photos.php
  193. disallow: /share.php
  194. disallow: /share/
  195. disallow: /sharer.php
  196. disallow: /sharer/
  197.  
  198. user-agent: msnbot
  199. disallow: /ajax/
  200. disallow: /album.php
  201. disallow: /checkpoint/
  202. disallow: /contact_importer/
  203. disallow: /dialog/
  204. disallow: /fbml/ajax/dialog/
  205. disallow: /feeds/
  206. disallow: /file_download.php
  207. disallow: /hashtag/
  208. disallow: /l.php
  209. disallow: /live/
  210. disallow: /moments_app/
  211. disallow: /p.php
  212. disallow: /photo.php
  213. disallow: /photos.php
  214. disallow: /share.php
  215. disallow: /share/
  216. disallow: /sharer.php
  217. disallow: /sharer/
  218.  
  219. user-agent: Naverbot
  220. disallow: /ajax/
  221. disallow: /album.php
  222. disallow: /checkpoint/
  223. disallow: /contact_importer/
  224. disallow: /dialog/
  225. disallow: /fbml/ajax/dialog/
  226. disallow: /feeds/
  227. disallow: /file_download.php
  228. disallow: /hashtag/
  229. disallow: /l.php
  230. disallow: /live/
  231. disallow: /moments_app/
  232. disallow: /p.php
  233. disallow: /photo.php
  234. disallow: /photos.php
  235. disallow: /share.php
  236. disallow: /share/
  237. disallow: /sharer.php
  238. disallow: /sharer/
  239.  
  240. user-agent: seznambot
  241. disallow: /ajax/
  242. disallow: /album.php
  243. disallow: /checkpoint/
  244. disallow: /contact_importer/
  245. disallow: /dialog/
  246. disallow: /fbml/ajax/dialog/
  247. disallow: /feeds/
  248. disallow: /file_download.php
  249. disallow: /hashtag/
  250. disallow: /l.php
  251. disallow: /live/
  252. disallow: /moments_app/
  253. disallow: /p.php
  254. disallow: /photo.php
  255. disallow: /photos.php
  256. disallow: /share.php
  257. disallow: /share/
  258. disallow: /sharer.php
  259. disallow: /sharer/
  260.  
  261. user-agent: Slurp
  262. disallow: /ajax/
  263. disallow: /album.php
  264. disallow: /checkpoint/
  265. disallow: /contact_importer/
  266. disallow: /dialog/
  267. disallow: /fbml/ajax/dialog/
  268. disallow: /feeds/
  269. disallow: /file_download.php
  270. disallow: /hashtag/
  271. disallow: /l.php
  272. disallow: /live/
  273. disallow: /moments_app/
  274. disallow: /p.php
  275. disallow: /photo.php
  276. disallow: /photos.php
  277. disallow: /share.php
  278. disallow: /share/
  279. disallow: /sharer.php
  280. disallow: /sharer/
  281.  
  282. user-agent: teoma
  283. disallow: /ajax/
  284. disallow: /album.php
  285. disallow: /checkpoint/
  286. disallow: /contact_importer/
  287. disallow: /dialog/
  288. disallow: /fbml/ajax/dialog/
  289. disallow: /feeds/
  290. disallow: /file_download.php
  291. disallow: /hashtag/
  292. disallow: /l.php
  293. disallow: /live/
  294. disallow: /moments_app/
  295. disallow: /p.php
  296. disallow: /photo.php
  297. disallow: /photos.php
  298. disallow: /share.php
  299. disallow: /share/
  300. disallow: /sharer.php
  301. disallow: /sharer/
  302.  
  303. user-agent: TelegramBot
  304. disallow: /ajax/
  305. disallow: /album.php
  306. disallow: /checkpoint/
  307. disallow: /contact_importer/
  308. disallow: /dialog/
  309. disallow: /fbml/ajax/dialog/
  310. disallow: /feeds/
  311. disallow: /file_download.php
  312. disallow: /hashtag/
  313. disallow: /l.php
  314. disallow: /live/
  315. disallow: /moments_app/
  316. disallow: /p.php
  317. disallow: /photo.php
  318. disallow: /photos.php
  319. disallow: /share.php
  320. disallow: /share/
  321. disallow: /sharer.php
  322. disallow: /sharer/
  323.  
  324. user-agent: Twitterbot
  325. disallow: /ajax/
  326. disallow: /album.php
  327. disallow: /checkpoint/
  328. disallow: /contact_importer/
  329. disallow: /dialog/
  330. disallow: /fbml/ajax/dialog/
  331. disallow: /feeds/
  332. disallow: /file_download.php
  333. disallow: /hashtag/
  334. disallow: /l.php
  335. disallow: /live/
  336. disallow: /moments_app/
  337. disallow: /p.php
  338. disallow: /photo.php
  339. disallow: /photos.php
  340. disallow: /share.php
  341. disallow: /share/
  342. disallow: /sharer.php
  343. disallow: /sharer/
  344.  
  345. user-agent: Yandex
  346. disallow: /ajax/
  347. disallow: /album.php
  348. disallow: /checkpoint/
  349. disallow: /contact_importer/
  350. disallow: /dialog/
  351. disallow: /fbml/ajax/dialog/
  352. disallow: /feeds/
  353. disallow: /file_download.php
  354. disallow: /hashtag/
  355. disallow: /l.php
  356. disallow: /live/
  357. disallow: /moments_app/
  358. disallow: /p.php
  359. disallow: /photo.php
  360. disallow: /photos.php
  361. disallow: /share.php
  362. disallow: /share/
  363. disallow: /sharer.php
  364. disallow: /sharer/
  365.  
  366. disallow: /ajax/
  367. disallow: /album.php
  368. disallow: /checkpoint/
  369. disallow: /contact_importer/
  370. disallow: /dialog/
  371. disallow: /fbml/ajax/dialog/
  372. disallow: /feeds/
  373. disallow: /file_download.php
  374. disallow: /hashtag/
  375. disallow: /l.php
  376. disallow: /live/
  377. disallow: /moments_app/
  378. disallow: /p.php
  379. disallow: /photo.php
  380. disallow: /photos.php
  381. disallow: /share.php
  382. disallow: /share/
  383. disallow: /sharer.php
  384. disallow: /sharer/
  385.  
  386. user-agent: Applebot
  387. Allow: /ajax/bootloader-endpoint/
  388. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  389. Allow: /safetycheck/
  390.  
  391. user-agent: baiduspider
  392. Allow: /ajax/bootloader-endpoint/
  393. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  394. Allow: /safetycheck/
  395.  
  396. user-agent: Bingbot
  397. Allow: /ajax/bootloader-endpoint/
  398. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  399. Allow: /safetycheck/
  400.  
  401. user-agent: Discordbot
  402. Allow: /*/videos/
  403. Allow: /ajax/bootloader-endpoint/
  404. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  405. Allow: /safetycheck/
  406. Allow: /watch/?v=*
  407.  
  408. user-agent: facebookexternalhit
  409. Allow: /*/videos/
  410. Allow: /ajax/bootloader-endpoint/
  411. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  412. Allow: /safetycheck/
  413. Allow: /watch/?v=*
  414.  
  415. user-agent: Googlebot
  416. Allow: /ajax/bootloader-endpoint/
  417. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  418. Allow: /safetycheck/
  419.  
  420. user-agent: ia_archiver
  421. Allow: /about/privacy
  422. Allow: /ajax/bootloader-endpoint/
  423. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  424. Allow: /full_data_use_policy
  425. Allow: /legal/terms
  426. Allow: /policy.php
  427. Allow: /safetycheck/
  428.  
  429. user-agent: LinkedInBot
  430. Allow: /*/videos/
  431. Allow: /ajax/bootloader-endpoint/
  432. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  433. Allow: /safetycheck/
  434. Allow: /watch/?v=*
  435.  
  436. user-agent: msnbot
  437. Allow: /ajax/bootloader-endpoint/
  438. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  439. Allow: /safetycheck/
  440.  
  441. user-agent: Naverbot
  442. Allow: /ajax/bootloader-endpoint/
  443. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  444. Allow: /safetycheck/
  445.  
  446. user-agent: seznambot
  447. Allow: /ajax/bootloader-endpoint/
  448. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  449. Allow: /safetycheck/
  450.  
  451. user-agent: Slurp
  452. Allow: /ajax/bootloader-endpoint/
  453. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  454. Allow: /safetycheck/
  455.  
  456. user-agent: teoma
  457. Allow: /ajax/bootloader-endpoint/
  458. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  459. Allow: /safetycheck/
  460.  
  461. user-agent: TelegramBot
  462. Allow: /*/videos/
  463. Allow: /ajax/bootloader-endpoint/
  464. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  465. Allow: /safetycheck/
  466. Allow: /watch/?v=*
  467.  
  468. user-agent: Twitterbot
  469. Allow: /ajax/bootloader-endpoint/
  470. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  471. Allow: /safetycheck/
  472.  
  473. user-agent: Yandex
  474. Allow: /ajax/bootloader-endpoint/
  475. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  476. Allow: /safetycheck/
  477.  
  478. Allow: /ajax/bootloader-endpoint/
  479. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  480. Allow: /safetycheck/
  481.  
  482.  
Parsed in 0.061 seconds