1 package LWP::UserAgent;
4 use vars qw(@ISA $VERSION);
6 require LWP::MemberMixin;
7 @ISA = qw(LWP::MemberMixin);
11 use HTTP::Response ();
20 if ($ENV{PERL_LWP_USE_HTTP_10}) {
21 require LWP::Protocol::http10;
22 LWP::Protocol::implementor('http', 'LWP::Protocol::http10');
24 require LWP::Protocol::https10;
25 LWP::Protocol::implementor('https', 'LWP::Protocol::https10');
33 # Check for common user mistake
34 Carp::croak("Options to LWP::UserAgent should be key/value pairs, not hash reference")
35 if ref($_[1]) eq 'HASH';
37 my($class, %cnf) = @_;
38 LWP::Debug::trace('()');
40 my $agent = delete $cnf{agent};
41 $agent = $class->_agent unless defined $agent;
43 my $from = delete $cnf{from};
44 my $timeout = delete $cnf{timeout};
45 $timeout = 3*60 unless defined $timeout;
46 my $use_eval = delete $cnf{use_eval};
47 $use_eval = 1 unless defined $use_eval;
48 my $parse_head = delete $cnf{parse_head};
49 $parse_head = 1 unless defined $parse_head;
50 my $show_progress = delete $cnf{show_progress};
51 my $max_size = delete $cnf{max_size};
52 my $max_redirect = delete $cnf{max_redirect};
53 $max_redirect = 7 unless defined $max_redirect;
54 my $env_proxy = delete $cnf{env_proxy};
56 my $cookie_jar = delete $cnf{cookie_jar};
57 my $conn_cache = delete $cnf{conn_cache};
58 my $keep_alive = delete $cnf{keep_alive};
60 Carp::croak("Can't mix conn_cache and keep_alive")
61 if $conn_cache && $keep_alive;
64 my $protocols_allowed = delete $cnf{protocols_allowed};
65 my $protocols_forbidden = delete $cnf{protocols_forbidden};
67 my $requests_redirectable = delete $cnf{requests_redirectable};
68 $requests_redirectable = ['GET', 'HEAD']
69 unless defined $requests_redirectable;
71 # Actually ""s are just as good as 0's, but for concision we'll just say:
72 Carp::croak("protocols_allowed has to be an arrayref or 0, not \"$protocols_allowed\"!")
73 if $protocols_allowed and ref($protocols_allowed) ne 'ARRAY';
74 Carp::croak("protocols_forbidden has to be an arrayref or 0, not \"$protocols_forbidden\"!")
75 if $protocols_forbidden and ref($protocols_forbidden) ne 'ARRAY';
76 Carp::croak("requests_redirectable has to be an arrayref or 0, not \"$requests_redirectable\"!")
77 if $requests_redirectable and ref($requests_redirectable) ne 'ARRAY';
81 Carp::carp("Unrecognized LWP::UserAgent options: @{[sort keys %cnf]}");
88 use_eval => $use_eval,
89 parse_head => $parse_head,
90 show_progress=> $show_progress,
91 max_size => $max_size,
92 max_redirect => $max_redirect,
95 protocols_allowed => $protocols_allowed,
96 protocols_forbidden => $protocols_forbidden,
97 requests_redirectable => $requests_redirectable,
100 $self->agent($agent) if $agent;
101 $self->cookie_jar($cookie_jar) if $cookie_jar;
102 $self->env_proxy if $env_proxy;
104 $self->protocols_allowed( $protocols_allowed ) if $protocols_allowed;
105 $self->protocols_forbidden($protocols_forbidden) if $protocols_forbidden;
108 $conn_cache ||= { total_capacity => $keep_alive };
110 $self->conn_cache($conn_cache) if $conn_cache;
116 # private method. check sanity of given $request
117 sub _request_sanity_check {
118 my($self, $request) = @_;
119 # some sanity checking
120 if (defined $request) {
122 Carp::croak("You need a request object, not a " . ref($request) . " object")
123 if ref($request) eq 'ARRAY' or ref($request) eq 'HASH' or
124 !$request->can('method') or !$request->can('uri');
127 Carp::croak("You need a request object, not '$request'");
131 Carp::croak("No request object passed in");
138 my($self, $request, $arg, $size) = @_;
139 $self->_request_sanity_check($request);
141 my($method, $url) = ($request->method, $request->uri);
143 local($SIG{__DIE__}); # protect against user defined die handlers
145 # Check that we have a METHOD and a URL first
146 return _new_response($request, &HTTP::Status::RC_BAD_REQUEST, "Method missing")
148 return _new_response($request, &HTTP::Status::RC_BAD_REQUEST, "URL missing")
150 return _new_response($request, &HTTP::Status::RC_BAD_REQUEST, "URL must be absolute")
153 LWP::Debug::trace("$method $url");
155 # Locate protocol to use
157 my $proxy = $self->_need_proxy($url);
158 if (defined $proxy) {
159 $scheme = $proxy->scheme;
162 $scheme = $url->scheme;
168 # Honor object-specific restrictions by forcing protocol objects
169 # into class LWP::Protocol::nogo.
171 if($x = $self->protocols_allowed) {
172 if(grep lc($_) eq $scheme, @$x) {
173 LWP::Debug::trace("$scheme URLs are among $self\'s allowed protocols (@$x)");
176 LWP::Debug::trace("$scheme URLs aren't among $self\'s allowed protocols (@$x)");
177 require LWP::Protocol::nogo;
178 $protocol = LWP::Protocol::nogo->new;
181 elsif ($x = $self->protocols_forbidden) {
182 if(grep lc($_) eq $scheme, @$x) {
183 LWP::Debug::trace("$scheme URLs are among $self\'s forbidden protocols (@$x)");
184 require LWP::Protocol::nogo;
185 $protocol = LWP::Protocol::nogo->new;
188 LWP::Debug::trace("$scheme URLs aren't among $self\'s forbidden protocols (@$x)");
191 # else fall thru and create the protocol object normally
195 $protocol = eval { LWP::Protocol::create($scheme, $self) };
197 $@ =~ s/ at .* line \d+.*//s; # remove file/line number
198 my $response = _new_response($request, &HTTP::Status::RC_NOT_IMPLEMENTED, $@);
199 if ($scheme eq "https") {
200 $response->message($response->message . " (Crypt::SSLeay not installed)");
201 $response->content_type("text/plain");
202 $response->content(<<EOT);
203 LWP will support https URLs if the Crypt::SSLeay module is installed.
204 More information at <http://www.linpro.no/lwp/libwww-perl/README.SSL>.
211 # Extract fields that will be used below
212 my ($timeout, $cookie_jar, $use_eval, $parse_head, $max_size) =
213 @{$self}{qw(timeout cookie_jar use_eval parse_head max_size)};
216 $self->progress("begin", $request);
218 # we eval, and turn dies into responses below
220 $response = $protocol->request($request, $proxy,
221 $arg, $size, $timeout);
224 $@ =~ s/ at .* line \d+.*//s; # remove file/line number
225 $response = _new_response($request,
226 &HTTP::Status::RC_INTERNAL_SERVER_ERROR,
231 $response = $protocol->request($request, $proxy,
232 $arg, $size, $timeout);
233 # XXX: Should we die unless $response->is_success ???
236 $response->request($request); # record request for reference
237 $cookie_jar->extract_cookies($response) if $cookie_jar;
238 $response->header("Client-Date" => HTTP::Date::time2str(time));
240 $self->progress("end", $response);
247 my($self, $request) = @_;
248 $self->_request_sanity_check($request);
250 # Extract fields that will be used below
251 my ($agent, $from, $cookie_jar, $max_size, $def_headers) =
252 @{$self}{qw(agent from cookie_jar max_size def_headers)};
254 # Set User-Agent and From headers if they are defined
255 $request->init_header('User-Agent' => $agent) if $agent;
256 $request->init_header('From' => $from) if $from;
257 if (defined $max_size) {
258 my $last = $max_size - 1;
259 $last = 0 if $last < 0; # there is no way to actually request no content
260 $request->init_header('Range' => "bytes=0-$last");
262 $cookie_jar->add_cookie_header($request) if $cookie_jar;
265 for my $h ($def_headers->header_field_names) {
266 $request->init_header($h => [$def_headers->header($h)]);
276 my($self, $request, $arg, $size) = @_;
277 $self->_request_sanity_check($request);
278 my $new_request = $self->prepare_request($request);
279 return($self->send_request($new_request, $arg, $size));
285 my($self, $request, $arg, $size, $previous) = @_;
287 LWP::Debug::trace('()');
289 my $response = $self->simple_request($request, $arg, $size);
291 my $code = $response->code;
292 $response->previous($previous) if defined $previous;
294 LWP::Debug::debug('Simple response: ' .
295 (HTTP::Status::status_message($code) ||
296 "Unknown code $code"));
298 if ($code == &HTTP::Status::RC_MOVED_PERMANENTLY or
299 $code == &HTTP::Status::RC_FOUND or
300 $code == &HTTP::Status::RC_SEE_OTHER or
301 $code == &HTTP::Status::RC_TEMPORARY_REDIRECT)
303 my $referral = $request->clone;
305 # These headers should never be forwarded
306 $referral->remove_header('Host', 'Cookie');
308 if ($referral->header('Referer') &&
309 $request->url->scheme eq 'https' &&
310 $referral->url->scheme eq 'http')
312 # RFC 2616, section 15.1.3.
313 LWP::Debug::trace("https -> http redirect, suppressing Referer");
314 $referral->remove_header('Referer');
317 if ($code == &HTTP::Status::RC_SEE_OTHER ||
318 $code == &HTTP::Status::RC_FOUND)
320 my $method = uc($referral->method);
321 unless ($method eq "GET" || $method eq "HEAD") {
322 $referral->method("GET");
323 $referral->content("");
324 $referral->remove_content_headers;
328 # And then we update the URL based on the Location:-header.
329 my $referral_uri = $response->header('Location');
331 # Some servers erroneously return a relative URL for redirects,
332 # so make it absolute if it not already is.
333 local $URI::ABS_ALLOW_RELATIVE_SCHEME = 1;
334 my $base = $response->base;
335 $referral_uri = "" unless defined $referral_uri;
336 $referral_uri = $HTTP::URI_CLASS->new($referral_uri, $base)
339 $referral->url($referral_uri);
341 # Check for loop in the redirects, we only count
345 if (++$count > $self->{max_redirect}) {
346 $response->header("Client-Warning" =>
347 "Redirect loop detected (max_redirect = $self->{max_redirect})");
353 return $response unless $self->redirect_ok($referral, $response);
354 return $self->request($referral, $arg, $size, $response);
357 elsif ($code == &HTTP::Status::RC_UNAUTHORIZED ||
358 $code == &HTTP::Status::RC_PROXY_AUTHENTICATION_REQUIRED
361 my $proxy = ($code == &HTTP::Status::RC_PROXY_AUTHENTICATION_REQUIRED);
362 my $ch_header = $proxy ? "Proxy-Authenticate" : "WWW-Authenticate";
363 my @challenge = $response->header($ch_header);
364 unless (@challenge) {
365 $response->header("Client-Warning" =>
366 "Missing Authenticate header");
370 require HTTP::Headers::Util;
371 CHALLENGE: for my $challenge (@challenge) {
372 $challenge =~ tr/,/;/; # "," is used to separate auth-params!!
373 ($challenge) = HTTP::Headers::Util::split_header_words($challenge);
374 my $scheme = lc(shift(@$challenge));
375 shift(@$challenge); # no value
376 $challenge = { @$challenge }; # make rest into a hash
377 for (keys %$challenge) { # make sure all keys are lower case
378 $challenge->{lc $_} = delete $challenge->{$_};
381 unless ($scheme =~ /^([a-z]+(?:-[a-z]+)*)$/) {
382 $response->header("Client-Warning" =>
383 "Bad authentication scheme '$scheme'");
386 $scheme = $1; # untainted now
387 my $class = "LWP::Authen::\u$scheme";
391 unless (%{"$class\::"}) {
393 eval "require $class";
395 if ($@ =~ /^Can\'t locate/) {
396 $response->header("Client-Warning" =>
397 "Unsupported authentication scheme '$scheme'");
400 $response->header("Client-Warning" => $@);
405 unless ($class->can("authenticate")) {
406 $response->header("Client-Warning" =>
407 "Unsupported authentication scheme '$scheme'");
410 return $class->authenticate($self, $proxy, $challenge, $response,
411 $request, $arg, $size);
420 # Now the shortcuts...
423 require HTTP::Request::Common;
424 my($self, @parameters) = @_;
425 my @suff = $self->_process_colonic_headers(\@parameters,1);
426 return $self->request( HTTP::Request::Common::GET( @parameters ), @suff );
431 require HTTP::Request::Common;
432 my($self, @parameters) = @_;
433 my @suff = $self->_process_colonic_headers(\@parameters, (ref($parameters[1]) ? 2 : 1));
434 return $self->request( HTTP::Request::Common::POST( @parameters ), @suff );
439 require HTTP::Request::Common;
440 my($self, @parameters) = @_;
441 my @suff = $self->_process_colonic_headers(\@parameters,1);
442 return $self->request( HTTP::Request::Common::HEAD( @parameters ), @suff );
446 sub _process_colonic_headers {
447 # Process :content_cb / :content_file / :read_size_hint headers.
448 my($self, $args, $start_index) = @_;
451 for(my $i = $start_index; $i < @$args; $i += 2) {
452 next unless defined $args->[$i];
454 #printf "Considering %s => %s\n", $args->[$i], $args->[$i + 1];
456 if($args->[$i] eq ':content_cb') {
457 # Some sanity-checking...
458 $arg = $args->[$i + 1];
459 Carp::croak("A :content_cb value can't be undef") unless defined $arg;
460 Carp::croak("A :content_cb value must be a coderef")
461 unless ref $arg and UNIVERSAL::isa($arg, 'CODE');
464 elsif ($args->[$i] eq ':content_file') {
465 $arg = $args->[$i + 1];
467 # Some sanity-checking...
468 Carp::croak("A :content_file value can't be undef")
470 Carp::croak("A :content_file value can't be a reference")
472 Carp::croak("A :content_file value can't be \"\"")
476 elsif ($args->[$i] eq ':read_size_hint') {
477 $size = $args->[$i + 1];
478 # Bother checking it?
484 splice @$args, $i, 2;
488 # And return a suitable suffix-list for request(REQ,...)
490 return unless defined $arg;
491 return $arg, $size if defined $size;
495 my @ANI = qw(- \ | /);
498 my($self, $status, $m) = @_;
499 return unless $self->{show_progress};
500 if ($status eq "begin") {
501 print STDERR "** ", $m->method, " ", $m->uri, " ==> ";
502 $self->{progress_start} = time;
503 $self->{progress_lastp} = "";
504 $self->{progress_ani} = 0;
506 elsif ($status eq "end") {
507 delete $self->{progress_lastp};
508 delete $self->{progress_ani};
509 print STDERR $m->status_line;
510 my $t = time - delete $self->{progress_start};
511 print STDERR " (${t}s)" if $t;
514 elsif ($status eq "tick") {
515 print STDERR "$ANI[$self->{progress_ani}++]\b";
516 $self->{progress_ani} %= @ANI;
519 my $p = sprintf "%3.0f%%", $status * 100;
520 return if $p eq $self->{progress_lastp};
521 print STDERR "$p\b\b\b\b";
522 $self->{progress_lastp} = $p;
529 # This whole allow/forbid thing is based on man 1 at's way of doing things.
531 sub is_protocol_supported
533 my($self, $scheme) = @_;
535 # assume we got a reference to an URI object
536 $scheme = $scheme->scheme;
539 Carp::croak("Illegal scheme '$scheme' passed to is_protocol_supported")
541 $scheme = lc $scheme;
545 if(ref($self) and $x = $self->protocols_allowed) {
546 return 0 unless grep lc($_) eq $scheme, @$x;
548 elsif (ref($self) and $x = $self->protocols_forbidden) {
549 return 0 if grep lc($_) eq $scheme, @$x;
552 local($SIG{__DIE__}); # protect against user defined die handlers
553 $x = LWP::Protocol::implementor($scheme);
554 return 1 if $x and $x ne 'LWP::Protocol::nogo';
559 sub protocols_allowed { shift->_elem('protocols_allowed' , @_) }
560 sub protocols_forbidden { shift->_elem('protocols_forbidden' , @_) }
561 sub requests_redirectable { shift->_elem('requests_redirectable', @_) }
566 # RFC 2616, section 10.3.2 and 10.3.3 say:
567 # If the 30[12] status code is received in response to a request other
568 # than GET or HEAD, the user agent MUST NOT automatically redirect the
569 # request unless it can be confirmed by the user, since this might
570 # change the conditions under which the request was issued.
572 # Note that this routine used to be just:
573 # return 0 if $_[1]->method eq "POST"; return 1;
575 my($self, $new_request, $response) = @_;
576 my $method = $response->request->method;
577 return 0 unless grep $_ eq $method,
578 @{ $self->requests_redirectable || [] };
580 if ($new_request->url->scheme eq 'file') {
581 $response->header("Client-Warning" =>
582 "Can't redirect to a file:// URL!");
586 # Otherwise it's apparently okay...
593 my($self, $netloc, $realm, $uid, $pass) = @_;
594 @{ $self->{'basic_authentication'}{lc($netloc)}{$realm} } =
599 sub get_basic_credentials
601 my($self, $realm, $uri, $proxy) = @_;
604 my $host_port = lc($uri->host_port);
605 if (exists $self->{'basic_authentication'}{$host_port}{$realm}) {
606 return @{ $self->{'basic_authentication'}{$host_port}{$realm} };
609 return (undef, undef);
615 my $old = $self->{agent};
618 $agent .= $self->_agent if $agent && $agent =~ /\s+$/;
619 $self->{agent} = $agent;
625 sub _agent { "libwww-perl/$LWP::VERSION" }
627 sub timeout { shift->_elem('timeout', @_); }
628 sub from { shift->_elem('from', @_); }
629 sub parse_head { shift->_elem('parse_head', @_); }
630 sub max_size { shift->_elem('max_size', @_); }
631 sub max_redirect { shift->_elem('max_redirect', @_); }
636 my $old = $self->{cookie_jar};
639 if (ref($jar) eq "HASH") {
640 require HTTP::Cookies;
641 $jar = HTTP::Cookies->new(%$jar);
643 $self->{cookie_jar} = $jar;
648 sub default_headers {
650 my $old = $self->{def_headers} ||= HTTP::Headers->new;
652 $self->{def_headers} = shift;
659 return $self->default_headers->header(@_);
665 my $old = $self->{conn_cache};
668 if (ref($cache) eq "HASH") {
669 require LWP::ConnCache;
670 $cache = LWP::ConnCache->new(%$cache);
672 $self->{conn_cache} = $cache;
679 sub use_eval { shift->_elem('use_eval', @_); }
682 Carp::carp("LWP::UserAgent->use_alarm(BOOL) is a no-op")
691 my $copy = bless { %$self }, ref $self; # copy most fields
693 # elements that are references must be handled in a special way
694 $copy->{'proxy'} = { %{$self->{'proxy'}} };
695 $copy->{'no_proxy'} = [ @{$self->{'no_proxy'}} ]; # copy array
697 # remove reference to objects for now
698 delete $copy->{cookie_jar};
699 delete $copy->{conn_cache};
707 my($self, $url, $file) = @_;
709 LWP::Debug::trace('()');
710 my $request = HTTP::Request->new('GET', $url);
713 my($mtime) = (stat($file))[9];
715 $request->header('If-Modified-Since' =>
716 HTTP::Date::time2str($mtime));
719 my $tmpfile = "$file-$$";
721 my $response = $self->request($request, $tmpfile);
722 if ($response->is_success) {
724 my $file_length = (stat($tmpfile))[7];
725 my($content_length) = $response->header('Content-length');
727 if (defined $content_length and $file_length < $content_length) {
729 die "Transfer truncated: " .
730 "only $file_length out of $content_length bytes received\n";
732 elsif (defined $content_length and $file_length > $content_length) {
734 die "Content-length mismatch: " .
735 "expected $content_length bytes, got $file_length\n";
740 # Some dosish systems fail to rename if the target exists
744 rename($tmpfile, $file) or
745 die "Cannot rename '$tmpfile' to '$file': $!\n";
747 if (my $lm = $response->last_modified) {
748 # make sure the file has the same last modification time
749 utime $lm, $lm, $file;
765 LWP::Debug::trace("$key @_");
767 return map $self->proxy($_, @_), @$key if ref $key;
769 my $old = $self->{'proxy'}{$key};
770 $self->{'proxy'}{$key} = shift if @_;
778 while(($k, $v) = each %ENV) {
779 if ($ENV{REQUEST_METHOD}) {
780 # Need to be careful when called in the CGI environment, as
781 # the HTTP_PROXY variable is under control of that other guy.
782 next if $k =~ /^HTTP_/;
783 $k = "HTTP_PROXY" if $k eq "CGI_HTTP_PROXY";
786 next unless $k =~ /^(.*)_proxy$/;
789 $self->no_proxy(split(/\s*,\s*/, $v));
792 $self->proxy($k, $v);
801 push(@{ $self->{'no_proxy'} }, @no);
804 $self->{'no_proxy'} = [];
809 # Private method which returns the URL of the Proxy configured for this
810 # URL, or undefined if none is configured.
813 my($self, $url) = @_;
814 $url = $HTTP::URI_CLASS->new($url) unless ref $url;
816 my $scheme = $url->scheme || return;
817 if (my $proxy = $self->{'proxy'}{$scheme}) {
818 if (@{ $self->{'no_proxy'} }) {
819 if (my $host = eval { $url->host }) {
820 for my $domain (@{ $self->{'no_proxy'} }) {
821 if ($host =~ /\Q$domain\E$/) {
822 LWP::Debug::trace("no_proxy configured");
828 LWP::Debug::debug("Proxied to $proxy");
829 return $HTTP::URI_CLASS->new($proxy);
831 LWP::Debug::debug('Not proxied');
837 my($request, $code, $message) = @_;
838 my $response = HTTP::Response->new($code, $message);
839 $response->request($request);
840 $response->header("Client-Date" => HTTP::Date::time2str(time));
841 $response->header("Client-Warning" => "Internal response");
842 $response->header("Content-Type" => "text/plain");
843 $response->content("$code $message\n");
854 LWP::UserAgent - Web user agent class
858 require LWP::UserAgent;
860 my $ua = LWP::UserAgent->new;
864 my $response = $ua->get('http://search.cpan.org/');
866 if ($response->is_success) {
867 print $response->content; # or whatever
870 die $response->status_line;
875 The C<LWP::UserAgent> is a class implementing a web user agent.
876 C<LWP::UserAgent> objects can be used to dispatch web requests.
878 In normal use the application creates an C<LWP::UserAgent> object, and
879 then configures it with values for timeouts, proxies, name, etc. It
880 then creates an instance of C<HTTP::Request> for the request that
881 needs to be performed. This request is then passed to one of the
882 request method the UserAgent, which dispatches it using the relevant
883 protocol, and returns a C<HTTP::Response> object. There are
884 convenience methods for sending the most common request types: get(),
885 head() and post(). When using these methods then the creation of the
886 request object is hidden as shown in the synopsis above.
888 The basic approach of the library is to use HTTP style communication
889 for all protocol schemes. This means that you will construct
890 C<HTTP::Request> objects and receive C<HTTP::Response> objects even
891 for non-HTTP resources like I<gopher> and I<ftp>. In order to achieve
892 even more similarity to HTTP style communications, gopher menus and
893 file directories are converted to HTML documents.
895 =head1 CONSTRUCTOR METHODS
897 The following constructor methods are available:
901 =item $ua = LWP::UserAgent->new( %options )
903 This method constructs a new C<LWP::UserAgent> object and returns it.
904 Key/value pair arguments may be provided to set up the initial state.
905 The following options correspond to attribute methods described below:
908 ----------- --------------------
909 agent "libwww-perl/#.##"
913 default_headers HTTP::Headers->new
917 protocols_allowed undef
918 protocols_forbidden undef
919 requests_redirectable ['GET', 'HEAD']
922 The following additional options are also accepted: If the
923 C<env_proxy> option is passed in with a TRUE value, then proxy
924 settings are read from environment variables (see env_proxy() method
925 below). If the C<keep_alive> option is passed in, then a
926 C<LWP::ConnCache> is set up (see conn_cache() method below). The
927 C<keep_alive> value is passed on as the C<total_capacity> for the
932 Returns a copy of the LWP::UserAgent object.
938 The settings of the configuration attributes modify the behaviour of the
939 C<LWP::UserAgent> when it dispatches requests. Most of these can also
940 be initialized by options passed to the constructor method.
942 The following attributes methods are provided. The attribute value is
943 left unchanged if no argument is given. The return value from each
944 method is the old attribute value.
950 =item $ua->agent( $product_id )
952 Get/set the product token that is used to identify the user agent on
953 the network. The agent value is sent as the "User-Agent" header in
954 the requests. The default is the string returned by the _agent()
957 If the $product_id ends with space then the _agent() string is
960 The user agent string should be one or more simple product identifiers
961 with an optional version number separated by the "/" character.
964 $ua->agent('Checkbot/0.4 ' . $ua->_agent);
965 $ua->agent('Checkbot/0.4 '); # same as above
966 $ua->agent('Mozilla/5.0');
967 $ua->agent(""); # don't identify
971 Returns the default agent identifier. This is a string of the form
972 "libwww-perl/#.##", where "#.##" is substituted with the version number
977 =item $ua->from( $email_address )
979 Get/set the e-mail address for the human user who controls
980 the requesting user agent. The address should be machine-usable, as
981 defined in RFC 822. The C<from> value is send as the "From" header in
982 the requests. Example:
984 $ua->from('gaas@cpan.org');
986 The default is to not send a "From" header. See the default_headers()
987 method for the more general interface that allow any header to be defaulted.
989 =item $ua->cookie_jar
991 =item $ua->cookie_jar( $cookie_jar_obj )
993 Get/set the cookie jar object to use. The only requirement is that
994 the cookie jar object must implement the extract_cookies($request) and
995 add_cookie_header($response) methods. These methods will then be
996 invoked by the user agent as requests are sent and responses are
997 received. Normally this will be a C<HTTP::Cookies> object or some
1000 The default is to have no cookie_jar, i.e. never automatically add
1001 "Cookie" headers to the requests.
1003 Shortcut: If a reference to a plain hash is passed in as the
1004 $cookie_jar_object, then it is replaced with an instance of
1005 C<HTTP::Cookies> that is initialized based on the hash. This form also
1006 automatically loads the C<HTTP::Cookies> module. It means that:
1008 $ua->cookie_jar({ file => "$ENV{HOME}/.cookies.txt" });
1010 is really just a shortcut for:
1012 require HTTP::Cookies;
1013 $ua->cookie_jar(HTTP::Cookies->new(file => "$ENV{HOME}/.cookies.txt"));
1015 =item $ua->default_headers
1017 =item $ua->default_headers( $headers_obj )
1019 Get/set the headers object that will provide default header values for
1020 any requests sent. By default this will be an empty C<HTTP::Headers>
1023 $ua->default_headers->push_header('Accept-Language' => "no, en");
1025 =item $ua->default_header( $field )
1027 =item $ua->default_header( $field => $value )
1029 This is just a short-cut for $ua->default_headers->header( $field =>
1032 $ua->default_header('Accept-Language' => "no, en");
1034 =item $ua->conn_cache
1036 =item $ua->conn_cache( $cache_obj )
1038 Get/set the C<LWP::ConnCache> object to use. See L<LWP::ConnCache>
1041 =item $ua->credentials( $netloc, $realm, $uname, $pass )
1043 Set the user name and password to be used for a realm. It is often more
1044 useful to specialize the get_basic_credentials() method instead.
1046 The $netloc a string of the form "<host>:<port>". The username and
1047 password will only be passed to this server. Example:
1049 $ua->credentials("www.example.com:80", "Some Realm", "foo", "secret");
1053 =item $ua->max_size( $bytes )
1055 Get/set the size limit for response content. The default is C<undef>,
1056 which means that there is no limit. If the returned response content
1057 is only partial, because the size limit was exceeded, then a
1058 "Client-Aborted" header will be added to the response. The content
1059 might end up longer than C<max_size> as we abort once appending a
1060 chunk of data makes the length exceed the limit. The "Content-Length"
1061 header, if present, will indicate the length of the full content and
1062 will normally not be the same as C<< length($res->content) >>.
1064 =item $ua->max_redirect
1066 =item $ua->max_redirect( $n )
1068 This reads or sets the object's limit of how many times it will obey
1069 redirection responses in a given request cycle.
1071 By default, the value is 7. This means that if you call request()
1072 method and the response is a redirect elsewhere which is in turn a
1073 redirect, and so on seven times, then LWP gives up after that seventh
1076 =item $ua->parse_head
1078 =item $ua->parse_head( $boolean )
1080 Get/set a value indicating whether we should initialize response
1081 headers from the E<lt>head> section of HTML documents. The default is
1082 TRUE. Do not turn this off, unless you know what you are doing.
1084 =item $ua->protocols_allowed
1086 =item $ua->protocols_allowed( \@protocols )
1088 This reads (or sets) this user agent's list of protocols that the
1089 request methods will exclusively allow. The protocol names are case
1092 For example: C<$ua-E<gt>protocols_allowed( [ 'http', 'https'] );>
1093 means that this user agent will I<allow only> those protocols,
1094 and attempts to use this user agent to access URLs with any other
1095 schemes (like "ftp://...") will result in a 500 error.
1097 To delete the list, call: C<$ua-E<gt>protocols_allowed(undef)>
1099 By default, an object has neither a C<protocols_allowed> list, nor a
1100 C<protocols_forbidden> list.
1102 Note that having a C<protocols_allowed> list causes any
1103 C<protocols_forbidden> list to be ignored.
1105 =item $ua->protocols_forbidden
1107 =item $ua->protocols_forbidden( \@protocols )
1109 This reads (or sets) this user agent's list of protocols that the
1110 request method will I<not> allow. The protocol names are case
1113 For example: C<$ua-E<gt>protocols_forbidden( [ 'file', 'mailto'] );>
1114 means that this user agent will I<not> allow those protocols, and
1115 attempts to use this user agent to access URLs with those schemes
1116 will result in a 500 error.
1118 To delete the list, call: C<$ua-E<gt>protocols_forbidden(undef)>
1120 =item $ua->requests_redirectable
1122 =item $ua->requests_redirectable( \@requests )
1124 This reads or sets the object's list of request names that
1125 C<$ua-E<gt>redirect_ok(...)> will allow redirection for. By
1126 default, this is C<['GET', 'HEAD']>, as per RFC 2616. To
1127 change to include 'POST', consider:
1129 push @{ $ua->requests_redirectable }, 'POST';
1133 =item $ua->timeout( $secs )
1135 Get/set the timeout value in seconds. The default timeout() value is
1136 180 seconds, i.e. 3 minutes.
1138 The requests is aborted if no activity on the connection to the server
1139 is observed for C<timeout> seconds. This means that the time it takes
1140 for the complete transaction and the request() method to actually
1141 return might be longer.
1145 =head2 Proxy attributes
1147 The following methods set up when requests should be passed via a
1152 =item $ua->proxy(\@schemes, $proxy_url)
1154 =item $ua->proxy($scheme, $proxy_url)
1156 Set/retrieve proxy URL for a scheme:
1158 $ua->proxy(['http', 'ftp'], 'http://proxy.sn.no:8001/');
1159 $ua->proxy('gopher', 'http://proxy.sn.no:8001/');
1161 The first form specifies that the URL is to be used for proxying of
1162 access methods listed in the list in the first method argument,
1163 i.e. 'http' and 'ftp'.
1165 The second form shows a shorthand form for specifying
1166 proxy URL for a single access scheme.
1168 =item $ua->no_proxy( $domain, ... )
1170 Do not proxy requests to the given domains. Calling no_proxy without
1171 any domains clears the list of domains. Eg:
1173 $ua->no_proxy('localhost', 'no', ...);
1175 =item $ua->env_proxy
1177 Load proxy settings from *_proxy environment variables. You might
1178 specify proxies like this (sh-syntax):
1180 gopher_proxy=http://proxy.my.place/
1181 wais_proxy=http://proxy.my.place/
1182 no_proxy="localhost,my.domain"
1183 export gopher_proxy wais_proxy no_proxy
1185 csh or tcsh users should use the C<setenv> command to define these
1186 environment variables.
1188 On systems with case insensitive environment variables there exists a
1189 name clash between the CGI environment variables and the C<HTTP_PROXY>
1190 environment variable normally picked up by env_proxy(). Because of
1191 this C<HTTP_PROXY> is not honored for CGI scripts. The
1192 C<CGI_HTTP_PROXY> environment variable can be used instead.
1196 =head1 REQUEST METHODS
1198 The methods described in this section are used to dispatch requests
1199 via the user agent. The following request methods are provided:
1203 =item $ua->get( $url )
1205 =item $ua->get( $url , $field_name => $value, ... )
1207 This method will dispatch a C<GET> request on the given $url. Further
1208 arguments can be given to initialize the headers of the request. These
1209 are given as separate name/value pairs. The return value is a
1210 response object. See L<HTTP::Response> for a description of the
1211 interface it provides.
1213 Fields names that start with ":" are special. These will not
1214 initialize headers of the request but will determine how the response
1215 content is treated. The following special field names are recognized:
1217 :content_file => $filename
1218 :content_cb => \&callback
1219 :read_size_hint => $bytes
1221 If a $filename is provided with the C<:content_file> option, then the
1222 response content will be saved here instead of in the response
1223 object. If a callback is provided with the C<:content_cb> option then
1224 this function will be called for each chunk of the response content as
1225 it is received from the server. If neither of these options are
1226 given, then the response content will accumulate in the response
1227 object itself. This might not be suitable for very large response
1228 bodies. Only one of C<:content_file> or C<:content_cb> can be
1229 specified. The content of unsuccessful responses will always
1230 accumulate in the response object itself, regardless of the
1231 C<:content_file> or C<:content_cb> options passed in.
1233 The C<:read_size_hint> option is passed to the protocol module which
1234 will try to read data from the server in chunks of this size. A
1235 smaller value for the C<:read_size_hint> will result in a higher
1236 number of callback invocations.
1238 The callback function is called with 3 arguments: a chunk of data, a
1239 reference to the response object, and a reference to the protocol
1240 object. The callback can abort the request by invoking die(). The
1241 exception message will show up as the "X-Died" header field in the
1242 response returned by the get() function.
1244 =item $ua->head( $url )
1246 =item $ua->head( $url , $field_name => $value, ... )
1248 This method will dispatch a C<HEAD> request on the given $url.
1249 Otherwise it works like the get() method described above.
1251 =item $ua->post( $url, \%form )
1253 =item $ua->post( $url, \@form )
1255 =item $ua->post( $url, \%form, $field_name => $value, ... )
1257 =item $ua->post( $url, $field_name => $value,... Content => \%form )
1259 =item $ua->post( $url, $field_name => $value,... Content => \@form )
1261 =item $ua->post( $url, $field_name => $value,... Content => $content )
1263 This method will dispatch a C<POST> request on the given $url, with
1264 %form or @form providing the key/value pairs for the fill-in form
1265 content. Additional headers and content options are the same as for
1268 This method will use the POST() function from C<HTTP::Request::Common>
1269 to build the request. See L<HTTP::Request::Common> for a details on
1270 how to pass form content and other advanced features.
1272 =item $ua->mirror( $url, $filename )
1274 This method will get the document identified by $url and store it in
1275 file called $filename. If the file already exists, then the request
1276 will contain an "If-Modified-Since" header matching the modification
1277 time of the file. If the document on the server has not changed since
1278 this time, then nothing happens. If the document has been updated, it
1279 will be downloaded again. The modification time of the file will be
1280 forced to match that of the server.
1282 The return value is the the response object.
1284 =item $ua->request( $request )
1286 =item $ua->request( $request, $content_file )
1288 =item $ua->request( $request, $content_cb )
1290 =item $ua->request( $request, $content_cb, $read_size_hint )
1292 This method will dispatch the given $request object. Normally this
1293 will be an instance of the C<HTTP::Request> class, but any object with
1294 a similar interface will do. The return value is a response object.
1295 See L<HTTP::Request> and L<HTTP::Response> for a description of the
1296 interface provided by these classes.
1298 The request() method will process redirects and authentication
1299 responses transparently. This means that it may actually send several
1300 simple requests via the simple_request() method described below.
1302 The request methods described above; get(), head(), post() and
1303 mirror(), will all dispatch the request they build via this method.
1304 They are convenience methods that simply hides the creation of the
1305 request object for you.
1307 The $content_file, $content_cb and $read_size_hint all correspond to
1308 options described with the get() method above.
1310 You are allowed to use a CODE reference as C<content> in the request
1311 object passed in. The C<content> function should return the content
1312 when called. The content can be returned in chunks. The content
1313 function will be invoked repeatedly until it return an empty string to
1314 signal that there is no more content.
1316 =item $ua->simple_request( $request )
1318 =item $ua->simple_request( $request, $content_file )
1320 =item $ua->simple_request( $request, $content_cb )
1322 =item $ua->simple_request( $request, $content_cb, $read_size_hint )
1324 This method dispatches a single request and returns the response
1325 received. Arguments are the same as for request() described above.
1327 The difference from request() is that simple_request() will not try to
1328 handle redirects or authentication responses. The request() method
1329 will in fact invoke this method for each simple request it sends.
1331 =item $ua->is_protocol_supported( $scheme )
1333 You can use this method to test whether this user agent object supports the
1334 specified C<scheme>. (The C<scheme> might be a string (like 'http' or
1335 'ftp') or it might be an URI object reference.)
1337 Whether a scheme is supported, is determined by the user agent's
1338 C<protocols_allowed> or C<protocols_forbidden> lists (if any), and by
1339 the capabilities of LWP. I.e., this will return TRUE only if LWP
1340 supports this protocol I<and> it's permitted for this particular
1345 =head2 Callback methods
1347 The following methods will be invoked as requests are processed. These
1348 methods are documented here because subclasses of C<LWP::UserAgent>
1349 might want to override their behaviour.
1353 =item $ua->prepare_request( $request )
1355 This method is invoked by simple_request(). Its task is to modify the
1356 given $request object by setting up various headers based on the
1357 attributes of the user agent. The return value should normally be the
1358 $request object passed in. If a different request object is returned
1359 it will be the one actually processed.
1361 The headers affected by the base implementation are; "User-Agent",
1362 "From", "Range" and "Cookie".
1364 =item $ua->redirect_ok( $prospective_request, $response )
1366 This method is called by request() before it tries to follow a
1367 redirection to the request in $response. This should return a TRUE
1368 value if this redirection is permissible. The $prospective_request
1369 will be the request to be sent if this method returns TRUE.
1371 The base implementation will return FALSE unless the method
1372 is in the object's C<requests_redirectable> list,
1373 FALSE if the proposed redirection is to a "file://..."
1374 URL, and TRUE otherwise.
1376 =item $ua->get_basic_credentials( $realm, $uri, $isproxy )
1378 This is called by request() to retrieve credentials for documents
1379 protected by Basic or Digest Authentication. The arguments passed in
1380 is the $realm provided by the server, the $uri requested and a boolean
1381 flag to indicate if this is authentication against a proxy server.
1383 The method should return a username and password. It should return an
1384 empty list to abort the authentication resolution attempt. Subclasses
1385 can override this method to prompt the user for the information. An
1386 example of this can be found in C<lwp-request> program distributed
1389 The base implementation simply checks a set of pre-stored member
1390 variables, set up with the credentials() method.
1392 =item $ua->progress( $status, $request_or_response )
1394 This is called frequently as the response is received regardless of
1395 how the content is processed. The method is called with $status
1396 "begin" at the start of processing the request and with $state "end"
1397 before the request method returns. In between these $status will be
1398 the fraction of the response currently received or the string "tick"
1399 if the fraction can't be calculated.
1401 When $status is "begin" the second argument is the request object,
1402 otherwise it is the response object.
1408 See L<LWP> for a complete overview of libwww-perl5. See L<lwpcook>
1409 and the scripts F<lwp-request> and F<lwp-download> for examples of
1412 See L<HTTP::Request> and L<HTTP::Response> for a description of the
1413 message objects dispatched and received. See L<HTTP::Request::Common>
1414 and L<HTML::Form> for other ways to build request objects.
1416 See L<WWW::Mechanize> and L<WWW::Search> for examples of more
1417 specialized user agents based on C<LWP::UserAgent>.
1421 Copyright 1995-2008 Gisle Aas.
1423 This library is free software; you can redistribute it and/or
1424 modify it under the same terms as Perl itself.