Analyze System Logs

🎯 Introduction to Logging

RHEL 9 uses two logging systems: systemd-journald (binary journal) and rsyslog (text files). Both work together to provide comprehensive system logging.

Logging Systems

Common Log Files (rsyslog)

Log Priority Levels

# From highest to lowest priority:
0 emerg    # System unusable
1 alert    # Immediate action required
2 crit     # Critical conditions
3 err      # Error conditions
4 warning  # Warning conditions
5 notice   # Normal but significant
6 info     # Informational messages
7 debug    # Debug messages

📜 journalctl - systemd Journal

Basic journalctl Commands

# View all journal entries (oldest first)
journalctl

# View in reverse (newest first)
journalctl -r

# Follow live logs (like tail -f)
journalctl -f

# Show only kernel messages
journalctl -k
# or
journalctl --dmesg

# Show N most recent entries
journalctl -n 50

# No paging (output all at once)
journalctl --no-pager

Filtering by Time

# Show logs since last boot
journalctl -b

# Show logs from previous boot
journalctl -b -1

# List all boots
journalctl --list-boots

# Since specific time
journalctl --since "2025-01-13 10:00:00"
journalctl --since "1 hour ago"
journalctl --since yesterday
journalctl --since today

# Until specific time
journalctl --until "2025-01-13 12:00:00"

# Between times
journalctl --since "2025-01-13 09:00" --until "2025-01-13 17:00"

Filtering by Service/Unit

# Show logs for specific service
journalctl -u sshd
journalctl -u httpd.service

# Multiple units
journalctl -u sshd -u httpd

# Since last boot for service
journalctl -u sshd -b

# Follow service logs
journalctl -u httpd -f

Filtering by Priority

# Show only errors and above (emerg, alert, crit, err)
journalctl -p err

# Show warnings and above
journalctl -p warning

# Show specific priority
journalctl -p 3  # err level

# Priority range
journalctl -p err -p warning

Filtering by Process/User

# Filter by PID
journalctl _PID=1234

# Filter by user ID
journalctl _UID=1000

# Filter by executable
journalctl _COMM=sshd

# Filter by systemd unit
journalctl _SYSTEMD_UNIT=httpd.service

Output Formats

# Detailed output (default)
journalctl -o verbose

# Short format (syslog style)
journalctl -o short

# JSON format
journalctl -o json

# JSON pretty-print
journalctl -o json-pretty

# Export format (for backup)
journalctl -o export

# Show messages only
journalctl -o cat

Combining Filters

# Service logs since last boot, errors only
journalctl -u sshd -b -p err

# Recent httpd errors
journalctl -u httpd -p err -n 20

# Today's authentication logs
journalctl -u sshd --since today -o short-precise

Disk Usage

# Show journal disk usage
journalctl --disk-usage

# Output example:
Archived and active journals take up 512.0M in the file system.

# Verify journal files
journalctl --verify

💾 Persistent Journal Storage

By default, systemd journal is stored in /run/log/journal (lost on reboot). Enable persistent storage to keep logs across reboots.

Enable Persistent Journal

# Create directory for persistent journal
sudo mkdir -p /var/log/journal

# Set ownership and permissions
sudo chown root:systemd-journal /var/log/journal
sudo chmod 2755 /var/log/journal

# Restart journald to use new location
sudo systemctl restart systemd-journald

# Or configure in /etc/systemd/journald.conf:
sudo vi /etc/systemd/journald.conf

[Journal]
Storage=persistent
SystemMaxUse=500M
SystemKeepFree=1G
MaxRetentionSec=1month

# Restart journald
sudo systemctl restart systemd-journald

Journal Configuration

Managing Journal Size

# Manually rotate journal
sudo journalctl --rotate

# Vacuum by size (keep only 500M)
sudo journalctl --vacuum-size=500M

# Vacuum by time (keep only 2 weeks)
sudo journalctl --vacuum-time=2weeks

# Vacuum by file count
sudo journalctl --vacuum-files=5

# Check space after vacuum
journalctl --disk-usage

📋 rsyslog - Traditional Logging

rsyslog receives logs from journald and writes them to text files in /var/log/.

rsyslog Configuration

# Main config file
/etc/rsyslog.conf

# Additional configs
/etc/rsyslog.d/*.conf

# Example configuration format:
facility.priority    action

# Examples:
*.info;mail.none;authpriv.none;cron.none    /var/log/messages
authpriv.*                                   /var/log/secure
mail.*                                       /var/log/maillog
cron.*                                       /var/log/cron

Facilities and Priorities

# Facilities:
auth, authpriv  # Security/authentication
cron            # Cron daemon
daemon          # System daemons
kern            # Kernel messages
mail            # Mail system
user            # User-level messages
local0-local7   # Custom applications

# Priority selectors:
*          # All priorities
none       # No priorities
.priority  # This priority and above
.=priority # Exactly this priority
.!priority # Not this priority

Viewing Traditional Logs

# View messages log
sudo less /var/log/messages
sudo tail -f /var/log/messages

# View authentication logs
sudo tail -f /var/log/secure

# Search for pattern
sudo grep "Failed password" /var/log/secure

# Last 20 lines
sudo tail -20 /var/log/messages

# Follow multiple logs
sudo tail -f /var/log/messages /var/log/secure

Custom rsyslog Rule

# Create custom log for httpd
sudo vi /etc/rsyslog.d/httpd.conf

# Add:
if $programname == 'httpd' then /var/log/httpd-custom.log
& stop

# Restart rsyslog
sudo systemctl restart rsyslog

# Verify
logger -t httpd "Test message"
sudo tail /var/log/httpd-custom.log

Remote Logging

# Configure client to send logs to remote server
sudo vi /etc/rsyslog.conf

# Add (UDP):
*.* @192.168.1.100:514

# Or (TCP):
*.* @@192.168.1.100:514

# Restart
sudo systemctl restart rsyslog

🔄 Log Rotation

logrotate manages log file rotation to prevent filling disk space.

logrotate Configuration

# Main config
/etc/logrotate.conf

# Service-specific configs
/etc/logrotate.d/

# Example config for custom log
sudo vi /etc/logrotate.d/myapp

/var/log/myapp/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    create 0640 root root
    sharedscripts
    postrotate
        systemctl reload myapp > /dev/null 2>&1 || true
    endscript
}

logrotate Directives

Manual Rotation

# Test rotation (dry run)
sudo logrotate -d /etc/logrotate.d/myapp

# Force rotation
sudo logrotate -f /etc/logrotate.d/myapp

# Rotate all logs
sudo logrotate /etc/logrotate.conf

# Check rotation status
cat /var/lib/logrotate/logrotate.status