######################################
# Running httpd on non-standard port #
######################################

Objective: Install, configure, and start httpd to run on port 8001

Basic steps:
- Install httpd
- Start and enable service
- Add port to firewall
- Update httpd config to listen on port 8001
- Add correct label for port 8001/tcp to default policy
- Start/restart httpd

Full solution:

# Install packages for apache (httpd)
[root@localhost ~]# yum group install "basic web server"

# start/enable httpd service, and  make hole in firewall for tcp/8001
[root@localhost ~]# systemctl enable httpd
ln -s '/usr/lib/systemd/system/httpd.service' '/etc/systemd/system/multi-user.target.wants/httpd.service'
[root@localhost ~]# systemctl start httpd
[root@localhost ~]# firewall-cmd --list-all
public (default, active)
  interfaces: enp0s3
  sources:
  services: dhcpv6-client ssh
  ports:
  masquerade: no
  forward-ports:
  icmp-blocks:
  rich rules:

[root@localhost ~]# firewall-cmd --permanent --add-port=8001/tcp
success
[root@localhost ~]# firewall-cmd --reload
success
[root@localhost ~]# firewall-cmd --list-all
public (default, active)
  interfaces: enp0s3
  sources:
  services: dhcpv6-client ssh
  ports: 8001/tcp
  masquerade: no
  forward-ports:
  icmp-blocks:
  rich rules:

# httpd is now listening on 80 and 443
[root@localhost ~]# ss -lntp | grep httpd
LISTEN     0      128                      :::80                      :::*      users:(("httpd",9724,4),("httpd",9723,4),("httpd",9722,4),("httpd",9721,4),("httpd",9720,4),("httpd",9719,4),("httpd",9718,4))
LISTEN     0      128                      :::443                     :::*      users:(("httpd",9724,6),("httpd",9723,6),("httpd",9722,6),("httpd",9721,6),("httpd",9720,6),("httpd",9719,6),("httpd",9718,6))

# Update listening port in httpd conf
# Change "Listen 80" to "Listen 8001"
[root@localhost ~]# vi /etc/httpd/conf/httpd.conf

# Attempt to reload the http config
[root@localhost ~]# systemctl reload httpd

# No ports listening anymore, httpd is now dead
[root@localhost ~]# ss -lntp | grep httpd

# SELinux logs are pretty vague
# HTTP service logs pretty much just say the service failed
[root@localhost ~]# systemctl status -l httpd
httpd.service - The Apache HTTP Server
   Loaded: loaded (/usr/lib/systemd/system/httpd.service; enabled)
   Active: failed (Result: exit-code) since Sat 2015-10-03 21:12:50 EDT; 11s ago
  Process: 10154 ExecStop=/bin/kill -WINCH ${MAINPID} (code=exited, status=1/FAILURE)
  Process: 10152 ExecReload=/usr/sbin/httpd $OPTIONS -k graceful (code=exited, status=0/SUCCESS)
  Process: 9718 ExecStart=/usr/sbin/httpd $OPTIONS -DFOREGROUND (code=exited, status=1/FAILURE)
 Main PID: 9718 (code=exited, status=1/FAILURE)
   Status: "Total requests: 0; Current requests/sec: 0; Current traffic:   0 B/sec"

Oct 03 21:08:44 localhost.localdomain systemd[1]: Starting The Apache HTTP Server...
Oct 03 21:08:44 localhost.localdomain httpd[9718]: AH00558: httpd: Could not reliably determine the server's fully qualified domain name, using localhost.localdomain. Set the 'ServerName' directive globally to suppress this message
Oct 03 21:08:44 localhost.localdomain systemd[1]: Started The Apache HTTP Server.
Oct 03 21:12:50 localhost.localdomain systemd[1]: Reloading The Apache HTTP Server.
Oct 03 21:12:50 localhost.localdomain httpd[10152]: AH00558: httpd: Could not reliably determine the server's fully qualified domain name, using localhost.localdomain. Set the 'ServerName' directive globally to suppress this message
Oct 03 21:12:50 localhost.localdomain systemd[1]: Reloaded The Apache HTTP Server.
Oct 03 21:12:50 localhost.localdomain systemd[1]: httpd.service: main process exited, code=exited, status=1/FAILURE
Oct 03 21:12:50 localhost.localdomain kill[10154]: kill: cannot find process ""
Oct 03 21:12:50 localhost.localdomain systemd[1]: httpd.service: control process exited, code=exited status=1
Oct 03 21:12:50 localhost.localdomain systemd[1]: Unit httpd.service entered failed state.

# /var/log/messages echoes the service logs
Oct  3 21:12:50 localhost systemd: httpd.service: main process exited, code=exited, status=1/FAILURE
Oct  3 21:12:50 localhost kill: kill: cannot find process ""
Oct  3 21:12:50 localhost systemd: httpd.service: control process exited, code=exited status=1
Oct  3 21:12:50 localhost systemd: Unit httpd.service entered failed state.

# /var/log/audit/audit.log at least gives a few hints
type=AVC msg=audit(1443921170.643:936): avc:  denied  { name_bind } for  pid=9718 comm="httpd" src=8001 scontext=system_u:system_r:httpd_t:s0 tcontext=system_u:object_r:unreserved_port_t:s0 tclass=tcp_socket

# Check what ports httpd is allowed to use, but wait no semanage command
[root@localhost ~]# semanage port -l | grep http_port_t
-bash: semanage: command not found

# What package is that from?
[root@localhost ~]# yum provides semanage
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
 * base: mirror.team-cymru.org
 * extras: mirror.team-cymru.org
 * updates: mirror.team-cymru.org
policycoreutils-python-2.2.5-15.el7.x86_64 : SELinux policy core python utilities
Repo        : base
Matched from:
Filename    : /usr/sbin/semanage

# Great, let's install it
[root@localhost ~]# yum install -y policycoreutils-python

# Now with semanage installed, check what ports httpd is allowed to use (not 8001)
[root@localhost ~]# semanage port -l | grep http_port_t
http_port_t                    tcp      80, 81, 443, 488, 8008, 8009, 8443, 9000

# Give port 8001/tcp the http_port_t label
[root@localhost ~]# semanage port -a -t http_port_t -p tcp 8001

# Verify it looks right
[root@localhost ~]# semanage port -l | grep http_port_t
http_port_t                    tcp      8001, 80, 81, 443, 488, 8008, 8009, 8443, 9000

# You can't just reaload httpd since it died before when you reloaded the config
[root@localhost ~]# systemctl reload httpd
Job for httpd.service failed. See 'systemctl status httpd.service' and 'journalctl -xn' for details.

# Start the httpd service
[root@localhost ~]# systemctl start httpd

# Verify that you see the ports you are expecting
[root@localhost ~]# ss -lntp | grep httpd
LISTEN     0      128                      :::8001                    :::*      users:(("httpd",10198,4),("httpd",10197,4),("httpd",10196,4),("httpd",10195,4),("httpd",10194,4),("httpd",10193,4),("httpd",10192,4))
LISTEN     0      128                      :::443                     :::*      users:(("httpd",10198,6),("httpd",10197,6),("httpd",10196,6),("httpd",10195,6),("httpd",10194,6),("httpd",10193,6),("httpd",10192,6))

# Access the httpd service (do it from another system to confirm firewall config)
[root@localhost ~]# curl http://localhost:8001

# Success!

# FYI: the httpd_t label is what restricts the ports for httpd processes
[root@localhost ~]# ps -efZ | grep http
system_u:system_r:httpd_t:s0    root     10192     1  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10193 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10194 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10195 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10196 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10197 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND
system_u:system_r:httpd_t:s0    apache   10198 10192  0 21:19 ?        00:00:00 /usr/sbin/httpd -DFOREGROUND