www/apache
1
0
Fork 0
mirror of https://github.com/apache/httpd.git synced 2026-10-23 22:49:01 +03:00
Code Activity

mod_mime_libmagic: New module, determining MIME types using libmagic(3):

Browse Source 
* modules/metadata/mod_mime_libmagic.c: New file.

* modules/metadata/config.m4: Add mime_libmagic.

* .github/workflows/linux.yml: Install libmagic-dev.

* docs/manual/mod/mod_mime_libmagic.xml: New file.

* docs/manual/mod/allmodules.xml: Add mod_mime_libmagic.

* changes-entries/mime-libmagic.txt: New file.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
GitHub: PR #752


git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1938247 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Joe Orton
2026-09-15 12:30:46 +00:00
parent b0d28c0a61
commit 2fc5be40ba
7 changed files with 543 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/linux.yml vendored
View File

@@ -404,7 +404,7 @@ jobs:
run: sudo apt-get install -o Acquire::Retries=5
libtool-bin libapr1-dev libaprutil1-dev
liblua5.3-dev libbrotli-dev libcurl4-openssl-dev
libnghttp2-dev libjansson-dev libpcre2-dev
libnghttp2-dev libjansson-dev libmagic-dev libpcre2-dev
${{ ! ( env.SKIP_TESTING || env.NO_TEST_FRAMEWORK ) && 'cpanminus perl-doc' || '' }}
${{ ! env.SKIP_TESTING && 'gdb' || '' }}
libsasl2-dev curl pipx ${{ matrix.pkgs }} check

2
changes-entries/mime-libmagic.txt Normal file
View File

@@ -0,0 +1,2 @@
*) mod_mime_libmagic: New module determining the MIME type of a file
from its contents using libmagic(3). [Joe Orton]

2
docs/log-message-tags/next-number
View File

@@ -1 +1 @@
10624
10629

1
docs/manual/mod/allmodules.xml
View File

@@ -81,6 +81,7 @@
<modulefile>mod_md.xml</modulefile>
<modulefile>mod_mime.xml</modulefile>
<modulefile>mod_mime_magic.xml</modulefile>
<modulefile>mod_mime_libmagic.xml</modulefile>
<modulefile>mod_negotiation.xml</modulefile>
<modulefile>mod_nw_ssl.xml</modulefile>
<modulefile>mod_policy.xml</modulefile>

186
docs/manual/mod/mod_mime_libmagic.xml Normal file
View File

@@ -0,0 +1,186 @@
<?xml version="1.0"?>
<!DOCTYPE modulesynopsis SYSTEM "../style/modulesynopsis.dtd">
<?xml-stylesheet type="text/xsl" href="../style/manual.en.xsl"?>
<!-- $LastChangedRevision$ -->
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<modulesynopsis metafile="mod_mime_libmagic.xml.meta">
<name>mod_mime_libmagic</name>
<description>Determines the MIME type of a file from its contents
using libmagic</description>
<status>Extension</status>
<sourcefile>mod_mime_libmagic.c</sourcefile>
<identifier>mime_libmagic_module</identifier>
<compatibility>Available in Apache 2.5.1 and later</compatibility>
<summary>
<p>This module determines the <glossary ref="mime-type">MIME
type</glossary> of a file by examining its contents, in the same
way the <code>file(1)</code> command does. It is intended as a
"second line of defense" for files which
<module>mod_mime</module> cannot type from their extension: it
runs after <module>mod_mime</module> and does nothing if a type
has already been set.</p>
<p>Unlike <module>mod_mime_magic</module>, which has its own
copy of the <code>file(1)</code> engine and its own magic
database, this module uses the system's <code>libmagic(3)</code>
library and magic database from the <code>file</code> package,
so the types it reports are those which <code>file --mime-type</code>
would report. The module is built only if libmagic is found at
configure time, and is only active in a server or virtual host
where <directive module="mod_mime_libmagic">MimeLibmagic</directive>
is <code>On</code>.</p>
<p>Where libmagic reports <code>application/octet-stream</code>
the module sets no type, leaving the response without a
<code>Content-Type</code> header, as <module>mod_mime_magic</module>
does for unrecognised content. Directories are typed as
<code>httpd/unix-directory</code> and empty files as
<code>text/plain</code>.</p>
<p>The module does not look inside compressed files and never
sets a <code>Content-Encoding</code>: a gzip file is reported as
<code>application/gzip</code>. This differs from
<module>mod_mime_magic</module>, which reports the compressed
file's encoding.</p>
</summary>
<seealso><module>mod_mime</module></seealso>
<seealso><module>mod_mime_magic</module></seealso>
<section id="threads"><title>Performance and memory overhead</title>
<p>A libmagic handle is created, loaded with the magic database
and destroyed for every request examined. Loading a compiled database
(<code>magic.mgc</code>) maps the file into memory rather than
parsing it, and the mapped pages are shared between all server
processes, so this is inexpensive. If the configured database
is a text file rather than a compiled one, it is parsed on every
request.</p>
<p>Only the first <directive module="mod_mime_libmagic"
>MimeLibmagicBytes</directive> bytes of a file are read and
examined, whatever its size.</p>
<p>Examining text content in particular can allocate a few
megabytes of memory temporarily. With a threaded MPM and a C
library which keeps a separate heap per thread, each server
thread retains its peak allocation, so the resident size of a
busy child process can grow by that much per thread before
levelling off. This is not a leak.</p>
</section>
<directivesynopsis>
<name>MimeLibmagic</name>
<description>Enable MIME type determination from file contents</description>
<syntax>MimeLibmagic On|Off</syntax>
<default>MimeLibmagic Off</default>
<contextlist><context>server config</context><context>virtual host</context>
</contextlist>
<usage>
<p>The <directive>MimeLibmagic</directive> directive enables the
module for a server or virtual host. Files which
<module>mod_mime</module> leaves untyped, typically those without
a recognised extension, then have their contents examined by
libmagic.</p>
<example><title>Example</title>
<highlight language="config">
MimeLibmagic On
</highlight>
</example>
</usage>
</directivesynopsis>
<directivesynopsis>
<name>MimeLibmagicFile</name>
<description>The magic database to use</description>
<syntax>MimeLibmagicFile <var>file-path</var></syntax>
<default>The default database of the installed libmagic</default>
<contextlist><context>server config</context><context>virtual host</context>
</contextlist>
<usage>
<p>The <directive>MimeLibmagicFile</directive> directive names
the magic database, in place of the default compiled into
libmagic (typically <code>/usr/share/misc/magic</code>, for which
the compiled <code>magic.mgc</code> alongside it is loaded).
The database must be in the format described in
<code>magic(5)</code>; the file distributed with
<module>mod_mime_magic</module> is not suitable. A relative path
is taken relative to the <directive module="core"
>ServerRoot</directive>. If the database cannot be loaded, an
error is logged for each request examined and the file is left
untyped.</p>
<p>Note that the <code>MAGIC</code> environment variable, if set
when the server is started, overrides the library default, but
not a database given with this directive.</p>
<example><title>Example</title>
<highlight language="config">
MimeLibmagicFile /usr/share/misc/magic.mgc
</highlight>
</example>
</usage>
</directivesynopsis>
<directivesynopsis>
<name>MimeLibmagicBytes</name>
<description>Number of bytes of a file examined</description>
<syntax>MimeLibmagicBytes <var>bytes</var></syntax>
<default>MimeLibmagicBytes 16384</default>
<contextlist><context>server config</context><context>virtual host</context>
</contextlist>
<usage>
<p>The <directive>MimeLibmagicBytes</directive> directive sets
how many bytes from the start of a file are read and examined.
Larger values allow magic rules which look deeper into a file to
match, at the cost of reading more of every examined file. The
maximum is 1048576.</p>
</usage>
</directivesynopsis>
<directivesynopsis>
<name>MimeLibmagicCharset</name>
<description>Add a charset parameter to text types</description>
<syntax>MimeLibmagicCharset On|Off</syntax>
<default>MimeLibmagicCharset Off</default>
<contextlist><context>server config</context><context>virtual host</context>
</contextlist>
<usage>
<p>When <directive>MimeLibmagicCharset</directive> is
<code>On</code>, the character encoding libmagic detects (such
as <code>us-ascii</code>, <code>utf-8</code> or
<code>iso-8859-1</code>) is added as a <code>charset</code>
parameter to <code>text/*</code> types, giving for example
<code>text/plain; charset=utf-8</code>. Other types are not
affected. As with <directive module="core"
>AddDefaultCharset</directive>, the detected charset should be
trusted only where the content is known to be encoded
consistently.</p>
</usage>
</directivesynopsis>
</modulesynopsis>

17
modules/metadata/config.m4
View File

@@ -6,6 +6,23 @@ APACHE_MODPATH_INIT(metadata)
APACHE_MODULE(env, clearing/setting of ENV vars, , , yes)
APACHE_MODULE(mime_magic, automagically determining MIME type)
APACHE_MODULE(mime_libmagic, determining MIME type using libmagic, , , most, [
ap_libmagic_found=no
if test -n "$PKGCONFIG"; then
AC_MSG_CHECKING([for libmagic via pkg-config])
if $PKGCONFIG --exists libmagic; then
ap_libmagic_found=yes
APR_ADDTO(MOD_CFLAGS, [`$PKGCONFIG --cflags libmagic`])
APR_ADDTO(MOD_MIME_LIBMAGIC_LDADD, [`$PKGCONFIG --libs libmagic`])
fi
AC_MSG_RESULT($ap_libmagic_found)
fi
if test "$ap_libmagic_found" = "no"; then
AC_MSG_WARN([libmagic not found])
enable_mime_libmagic=no
fi
])
APACHE_MODULE(cern_meta, CERN-type meta files, , , no)
APACHE_MODULE(expires, Expires header control, , , most)
APACHE_MODULE(headers, HTTP header control, , , yes)

335
modules/metadata/mod_mime_libmagic.c Normal file
View File

@@ -0,0 +1,335 @@
/* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* mod_mime_libmagic: determine the Content-Type of a file from its
* contents using libmagic(3) from the "file" package. A fallback for
* files which mod_mime cannot type by extension, in the style of
* mod_mime_magic, but using the system magic database and matching
* engine rather than a private copy.
*
* A magic_t handle is opened, loaded and closed for each request: all
* libmagic's mutable state lives in the handle, so this is safe from
* any number of threads, and loading a compiled (.mgc) database is an
* mmap() of a file already in the page cache.
*/
#include "apr.h"
#include "apr_strings.h"
#include "apr_lib.h"
#include <errno.h>
#include "ap_config.h"
#include "httpd.h"
#include "http_config.h"
#include "http_core.h"
#include "http_log.h"
#include "http_protocol.h"
#include "http_request.h"
#include <magic.h>
module AP_MODULE_DECLARE_DATA mime_libmagic_module;
#define UNSET (-1)
#define DEFAULT_BYTES 16384
#define MAX_BYTES 1048576
typedef struct {
int enabled; /* MimeLibmagic On|Off */
const char *magicfile; /* MimeLibmagicFile, or the library default */
apr_int64_t bytes; /* MimeLibmagicBytes */
int charset; /* MimeLibmagicCharset On|Off */
} libmagic_server_conf;
static void *create_server_conf(apr_pool_t *p, server_rec *s)
{
libmagic_server_conf *conf = apr_pcalloc(p, sizeof *conf);
conf->enabled = UNSET;
conf->magicfile = NULL;
conf->bytes = UNSET;
conf->charset = UNSET;
return conf;
}
static void *merge_server_conf(apr_pool_t *p, void *basev, void *addv)
{
libmagic_server_conf *base = basev, *add = addv;
libmagic_server_conf *conf = apr_palloc(p, sizeof *conf);
conf->enabled = add->enabled != UNSET ? add->enabled : base->enabled;
conf->magicfile = add->magicfile ? add->magicfile : base->magicfile;
conf->bytes = add->bytes != UNSET ? add->bytes : base->bytes;
conf->charset = add->charset != UNSET ? add->charset : base->charset;
return conf;
}
static const char *set_enabled(cmd_parms *cmd, void *dummy, int flag)
{
libmagic_server_conf *conf =
ap_get_module_config(cmd->server->module_config, &mime_libmagic_module);
conf->enabled = flag;
return NULL;
}
static const char *set_magicfile(cmd_parms *cmd, void *dummy, const char *arg)
{
libmagic_server_conf *conf =
ap_get_module_config(cmd->server->module_config, &mime_libmagic_module);
conf->magicfile = ap_server_root_relative(cmd->pool, arg);
if (!conf->magicfile) {
return apr_pstrcat(cmd->pool, "Invalid MimeLibmagicFile path ", arg, NULL);
}
return NULL;
}
static const char *set_bytes(cmd_parms *cmd, void *dummy, const char *arg)
{
libmagic_server_conf *conf =
ap_get_module_config(cmd->server->module_config, &mime_libmagic_module);
char *end;
apr_int64_t n = apr_strtoi64(arg, &end, 10);
if (*arg == '\0' || *end != '\0' || n < 1 || n > MAX_BYTES) {
return "MimeLibmagicBytes must be between 1 and "
APR_STRINGIFY(MAX_BYTES);
}
conf->bytes = n;
return NULL;
}
static const char *set_charset(cmd_parms *cmd, void *dummy, int flag)
{
libmagic_server_conf *conf =
ap_get_module_config(cmd->server->module_config, &mime_libmagic_module);
conf->charset = flag;
return NULL;
}
/* Open a handle with flags and load the database at path. Returns
* NULL after logging the failure. */
static magic_t open_magic(request_rec *r, int flags, const char *path)
{
magic_t ms = magic_open(flags);
if (!ms) {
ap_log_rerror(APLOG_MARK, APLOG_ERR, APR_FROM_OS_ERROR(errno), r,
APLOGNO(10624) "libmagic: could not open handle");
return NULL;
}
if (magic_load(ms, path) != 0) {
const char *err = magic_error(ms);
ap_log_rerror(APLOG_MARK, APLOG_ERR, APR_FROM_OS_ERROR(magic_errno(ms)),
r, APLOGNO(10625) "libmagic: could not load magic "
"database %s: %s", path, err ? err : "unknown error");
magic_close(ms);
return NULL;
}
return ms;
}
/* Set the content type from libmagic's result string, either "type"
* or "type; charset=encoding". Returns OK or DECLINED. */
static int set_type(request_rec *r, const char *result, int charset_on)
{
char *type = apr_pstrdup(r->pool, result);
char *charset = NULL;
const char *slash, *p;
char *sep;
sep = strchr(type, ';');
if (sep) {
*sep++ = '\0';
while (*sep == ' ') {
sep++;
}
if (strncmp(sep, "charset=", 8) == 0) {
charset = sep + 8;
}
}
/* Some matchers append a note, e.g. "application/x-executable, no
* program header": keep the type. */
type[strcspn(type, ", \t")] = '\0';
/* the type must be of the form token "/" token */
slash = ap_scan_http_token(type);
if (slash == type || *slash != '/'
|| (p = ap_scan_http_token(slash + 1)) == slash + 1 || *p != '\0') {
ap_log_rerror(APLOG_MARK, APLOG_WARNING, 0, r, APLOGNO(10626)
"libmagic: ignoring invalid type '%s' for %s",
result, r->filename);
return DECLINED;
}
ap_content_type_tolower(type);
/* Unknown binary content is left untyped, as mod_mime_magic does. */
if (strcmp(type, "application/octet-stream") == 0
|| strcmp(type, "application/x-empty") == 0) {
ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r,
"libmagic: no type for %s (%s)", r->filename, result);
return DECLINED;
}
if (charset_on && charset && strncmp(type, "text/", 5) == 0
&& strcmp(charset, "binary") != 0
&& strcmp(charset, "unknown-8bit") != 0
&& *charset && *ap_scan_http_token(charset) == '\0') {
type = apr_pstrcat(r->pool, type, "; charset=", charset, NULL);
}
ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r,
"libmagic: %s is '%s' (%s)", r->filename, type, result);
ap_set_content_type_ex(r, type, 1);
return OK;
}
static int libmagic_find_ct(request_rec *r)
{
const libmagic_server_conf *conf;
apr_file_t *fd;
apr_status_t rv;
apr_size_t nbytes, bytes;
magic_t ms;
const char *result, *path;
char *buf;
int ret, flags;
/* the file has to exist */
if (r->finfo.filetype == APR_NOFILE || !r->filename) {
return DECLINED;
}
/* was someone else already here? */
if (r->content_type) {
return DECLINED;
}
conf = ap_get_module_config(r->server->module_config, &mime_libmagic_module);
if (conf->enabled != 1) {
return DECLINED;
}
switch (r->finfo.filetype) {
case APR_DIR:
ap_set_content_type_ex(r, DIR_MAGIC_TYPE, 1);
return OK;
case APR_CHR:
case APR_BLK:
case APR_PIPE:
case APR_SOCK:
ap_set_content_type_ex(r, "application/octet-stream", 1);
return OK;
case APR_REG:
break;
default:
/* a broken symlink, or something exotic: leave it to the handler */
return DECLINED;
}
rv = apr_file_open(&fd, r->filename, APR_READ | APR_BINARY,
APR_OS_DEFAULT, r->pool);
if (rv != APR_SUCCESS) {
ap_log_rerror(APLOG_MARK, APLOG_DEBUG, rv, r,
"libmagic: could not open %s", r->filename);
return DECLINED;
}
bytes = conf->bytes != UNSET ? conf->bytes : DEFAULT_BYTES;
buf = apr_palloc(r->pool, bytes);
rv = apr_file_read_full(fd, buf, bytes, &nbytes);
apr_file_close(fd);
if (rv != APR_SUCCESS && !APR_STATUS_IS_EOF(rv)) {
ap_log_rerror(APLOG_MARK, APLOG_WARNING, rv, r, APLOGNO(10627)
"libmagic: could not read %s", r->filename);
return DECLINED;
}
if (nbytes == 0) {
ap_set_content_type_ex(r, "text/plain", 1);
return OK;
}
/* With no MimeLibmagicFile, the library's default database; action
* 1 means the user's ~/.magic is not consulted, unlike
* magic_load(ms, NULL). */
path = conf->magicfile ? conf->magicfile : magic_getpath(NULL, 1);
/* Never look inside compressed files: that path spawns external
* decompressors from the calling thread. */
flags = MAGIC_MIME_TYPE | MAGIC_NO_CHECK_COMPRESS
| (conf->charset == 1 ? MAGIC_MIME_ENCODING : 0);
ms = open_magic(r, flags, path);
if (!ms) {
return DECLINED;
}
result = magic_buffer(ms, buf, nbytes);
if (!result) {
const char *err = magic_error(ms);
ap_log_rerror(APLOG_MARK, APLOG_WARNING, APR_FROM_OS_ERROR(magic_errno(ms)),
r, APLOGNO(10628) "libmagic: could not determine type "
"of %s: %s", r->filename, err ? err : "unknown error");
ret = DECLINED;
}
else {
/* result points into the handle, so must be used before close */
ret = set_type(r, result, conf->charset == 1);
}
magic_close(ms);
return ret;
}
static const command_rec libmagic_cmds[] = {
AP_INIT_FLAG("MimeLibmagic", set_enabled, NULL, RSRC_CONF,
"Enable content type detection using libmagic"),
AP_INIT_TAKE1("MimeLibmagicFile", set_magicfile, NULL, RSRC_CONF,
"Path to the magic database, in place of the library default"),
AP_INIT_TAKE1("MimeLibmagicBytes", set_bytes, NULL, RSRC_CONF,
"Number of bytes of a file examined"),
AP_INIT_FLAG("MimeLibmagicCharset", set_charset, NULL, RSRC_CONF,
"Add a charset parameter to text types"),
{NULL}
};
static void register_hooks(apr_pool_t *p)
{
static const char * const aszPre[] = { "mod_mime.c", "mod_mime_magic.c", NULL };
/* run after mod_mime (and mod_mime_magic, if loaded) */
ap_hook_type_checker(libmagic_find_ct, aszPre, NULL, APR_HOOK_MIDDLE);
}
AP_DECLARE_MODULE(mime_libmagic) =
{
STANDARD20_MODULE_STUFF,
NULL, /* dir config creator */
NULL, /* dir merger --- default is to override */
create_server_conf, /* server config */
merge_server_conf, /* merge server config */
libmagic_cmds, /* command apr_table_t */
register_hooks /* register hooks */
};