0021290ef0e761fb1fdf1f8bf0d21bb0 *DESCRIPTION
243cfc67b7b06ed97d9ffd2a858cc964 *LICENSE
a7fb1820d76741eb1b7071d545446341 *NAMESPACE
7d8907fa236badf09136904b8798bf23 *NEWS.md
75242f1d73923cc7aa1860cdbfdaa8cf *R/get_robotstxt.R
bb8eb2d72245392ffde1e0d91f9dbedd *R/get_robotstxt_http_get.R
e57345372231ac5573a554d205544ad9 *R/guess_domain.R
ad57b0dfc971640f22f80d92abd65ab1 *R/parse_robotstxt.R
b276d748728b879ead6e6003bafbf1b2 *R/path_allowed.R
b77bd2cac2d1770bf68b0e9fa0cfc0a5 *R/paths_allowed.R
e002670688bb09fd0d464f674968fc65 *R/pipe.R
3df4af40a6b8b0073c2c913c2aea1afd *R/print_robotstxt.R
744a71103d4ea2137f919b9bc57962fe *R/print_robotstxt_text.R
1dd2e720f2caba61d07f8605315a6149 *R/remove_domain.R
06dc62482d729197a7e4aef9353d2329 *R/robotstxt.R
7d34b0e8417b244318a5e39c5ae010ec *R/rt_cache.R
716845e77f07f6f8f91132581ab79845 *R/rt_get_comments.R
6a0b9aa3b2b5571a8714ab712cb3d0db *R/rt_get_fields.R
ec72b13ccc25816f202e1fd05a25a4fc *R/rt_get_fields_worker.R
40d44e8efb0e17bc3daebdc2bb93dd2c *R/rt_get_useragent.R
dd0f9f184bba5559b9ebf606b66a57e7 *R/sanitize_path.R
2fd3a60fbacd2fa824ac672b8ac5de6c *R/sanitize_permission_values.R
ddafc5765a4a68b0d9c3e7625ec267bd *R/sanitize_permissions.R
da11d5b8a43c8abf18fcc36518745baa *R/tools.R
90c922eea1a5a03133a9442c4f557422 *R/valid_robotstxt.R
030aaf66ce47a6391735b189b911cff5 *README.md
07aaef12ef41f5835f0da7998fb3d100 *build/vignette.rds
4bfde16a5fa9f87374ea46998a23c43b *inst/doc/using_robotstxt.R
8ea285b77b146385e74129e581eaff94 *inst/doc/using_robotstxt.Rmd
c48cc57bf9d35d604e62553aa363258b *inst/doc/using_robotstxt.html
4d8921c3dc225aa07cd9b3f66cde3aff *inst/robotstxts/allow_single_bot.txt
e2b27c9301b4f24c3b2db3f8886b2dc1 *inst/robotstxts/crawl_delay.txt
8cb7a8b11cc42f1f77aa0670c43071a3 *inst/robotstxts/disallow_all_for_BadBot.txt
d35cb09d5662e686347b00b959c1550f *inst/robotstxts/disallow_all_for_all.txt
542398cc4e150fd288cea549bdca16a4 *inst/robotstxts/disallow_some_for_all.txt
9ca8636f08a496a82ed0a64694d005c8 *inst/robotstxts/disallow_two_at_once.txt
7215ee9c7d9dc229d2921a40e899ec5f *inst/robotstxts/empty.txt
d034a6e564902694ac6b372940786d80 *inst/robotstxts/host.txt
95d8d7433b4cd37b5630cf68500127f0 *inst/robotstxts/robots_amazon.txt
a09430767142d30f771f9726196048e1 *inst/robotstxts/robots_bundestag.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc2.txt
3b1a4b952779358ff7d6015b0dee0e68 *inst/robotstxts/robots_facebook.txt
c51eaab2c346b3a156c72097e52b10d9 *inst/robotstxts/robots_facebook_unsupported.txt
62dbaccb94d6ab8aa5e178339efbe127 *inst/robotstxts/robots_google.txt
658622dc6bda3645b94e37fa1ea38295 *inst/robotstxts/robots_new_york_times.txt
48a1701be03cf9590a8ef623c5f2e8e6 *inst/robotstxts/robots_pmeissner.txt
67dfdd8b9acd9363417c145388315dab *inst/robotstxts/robots_spiegel.txt
6ed07c3b696c3b4d47e1b3b4a42f5c7f *inst/robotstxts/robots_wikipedia.txt
b03c30fd553878fce698924ee1eca06e *inst/robotstxts/robots_wikipedia_20170706.txt
4637c3108489f0f0e5e283ef16884baf *inst/robotstxts/robots_yahoo.txt
6b8539213a6199601ee9d81c70a6ac31 *inst/robotstxts/selfhtml_Example.txt
2b9b53034510e3b66af8968145fe0a4f *inst/robotstxts/testing_comments.txt
b9108b9d14c259b287400d48d488a998 *man/get_robotstxt.Rd
771afecdcdc673681523159ce2358b5d *man/get_robotstxt_http_get.Rd
1bc0dd9463de688f6dc5022b8408fa4f *man/guess_domain.Rd
6a58cbd3fec1c4a180af5e76794305fc *man/is_valid_robotstxt.Rd
bb2cf9ca030a8efab23b1b232f980400 *man/named_list.Rd
d785fba36311037b573908948b3c107c *man/parse_robotstxt.Rd
8b9eae7b558e2feda8441cc213d49c0b *man/path_allowed.Rd
ca56c5e1dfbffadfdeabc6fb56531d61 *man/paths_allowed.Rd
21dcf150608efd6b317d5560114731e3 *man/pipe.Rd
07513817e6f91a93f67ffcc94e689e5b *man/print.robotstxt.Rd
cd914240910650636f6ec6524e5a223f *man/print.robotstxt_text.Rd
af3c208672ffce7a5911e8830a7facd7 *man/remove_domain.Rd
3c6e4dcfeb8d1ab23e971f1d78cf3191 *man/robotstxt.Rd
829dbc0e7dc29c20a176218e8561704a *man/rt_cache.Rd
21e08d07c661487ad4c1f2a67a2c765c *man/rt_get_comments.Rd
3fb293f87ace37db20f97081469edb19 *man/rt_get_fields.Rd
bebcba122f4ccf2d324dd88a475abb8a *man/rt_get_fields_worker.Rd
252430a070d01ed98b385ec8fd38f212 *man/rt_get_rtxt.Rd
10b6a9bf4ed2c9e829d15ff01a768153 *man/rt_get_useragent.Rd
f43aa8ad76ab5c2b2ea2737d5421cfce *man/rt_list_rtxt.Rd
aca99f66c8046ae574c13ebeef6b65e2 *man/sanitize_path.Rd
972577fd362db928cbdc0ff820b28bac *man/sanitize_permission_values.Rd
5824dd4fd622d4896eb17c366cfdc076 *man/sanitize_permissions.Rd
480e44e1c814faf1886f057b8a375590 *tests/testthat.R
8ae85233dc20826cc031b030ac47d512 *tests/testthat/test_parser.R
b2deab794122c9d15475d9fe5e385afc *tests/testthat/test_permissions.R
f1c2f190f8e683fb4784b864dd930bdc *tests/testthat/test_robotstxt.R
8ea285b77b146385e74129e581eaff94 *vignettes/using_robotstxt.Rmd
