User Tools

Site Tools


Sidebar

software:performance_co-pilot:compiling

Compiling on RHEL7

  • install rhel7.5 x86_64
# activate base and optional channels

yum install \
  https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm

yum -y install git flex python-devel make tar \
  bison perl-ExtUtils-MakeMaker.noarch gcc-c++
  
# ensure pcp user/group exist.
# 'make install' relies on this
useradd pcp

mkdir /pcpserv && cd /pcpserv
git clone https://github.com/performancecopilot/pcp
cd pcp
./Makepkgs
cd ./pcp-*/build/tar
./preinstall 
here=`pwd`
( cd /; tar xpf $here/pcp-*-[0-9]*.tar.gz )
./postinstall
systemctl start pmcd
systemctl status pmcd
pminfo 

# ./configure --prefix=/
# make
# make install

Compiling on RHEL8

  • install RHEL8.2
yum -y install git flex make tar \
  bison perl-ExtUtils-MakeMaker.noarch gcc-c++ \
  python3-devel rpm-build initscripts ncurses-devel \
  readline-devel zlib-devel autoconf systemd-devel
 
# libuv-devel needs to be installed, so pmproxy can
# communicate with grafana.
  
# ensure pcp user/group exist.
# 'make install' relies on this
useradd pcp

mkdir /pcpserv && cd /pcpserv
git clone https://github.com/performancecopilot/pcp
cd pcp
./Makepkgs
cd ./pcp-5*/build/rpm

# install rpms, i.e.
rpm -ivh pcp-5.*.x86_64.rpm pcp-libs-5.3.1-1.x86_64.rpm \
  pcp-conf-5.3.1-1.x86_64.rpm pcp-zeroconf-5.3.1-1.x86_64.rpm \
  pcp-doc-5.3.1-1.noarch.rpm pcp-pmda-dm-5.3.1-1.x86_64.rpm \
  pcp-pmda-nfsclient-5.3.1-1.x86_64.rpm \
  pcp-system-tools-5.3.1-1.x86_64.rpm \
  pcp-pmda-netcheck-5.3.1-1.x86_64.rpm  python3-pcp-5.3.1-1.x86_64.rpm

systemctl start pmcd
systemctl status pmcd
pminfo 

Compiling on Fedora34

  • install Fedora34
yum -y install git flex python-devel make tar bison gcc-c++ \
  perl-ExtUtils-MakeMaker.noarch gcc-c++ python3-devel rpm-build \
  initscripts ncurses-devel readline-devel zlib-devel \
  systemd-devel autoconf python2-devel libselinux-devel \
  setools-console python2-setuptools libsemanage-devel \
  selinux-policy-devel
  
# ensure pcp user/group exist.
# 'make install' relies on this
useradd pcp

mkdir /pcpserv && cd /pcpserv
git clone https://github.com/performancecopilot/pcp
cd pcp
./Makepkgs
cd ./pcp-5*/build/rpm

<install rpms>

systemctl start pmcd
systemctl status pmcd
pminfo 

Compiling on Debian Bullseye

# libuv1-dev is important for pmseries binary/redis
apt-get install git dpkg-dev libpython3.7-dev debhelper libreadline-dev \
  chrpath python-all python3-all python3-dev libnspr4-dev libnss3-dev \
  libmicrohttpd-dev libavahi-common-dev libclass-dbi-perl \
  libdbd-mysql-perl python3-psycopg2 dh-python libcairo2-dev \
  libextutils-autoinstall-perl libxml-tokeparser-perl libjson-perl \
  libnet-snmp-perl libnss3-tools libuv1-dev libsystemd-dev

# ensure pcp user/group exist.
# 'make install' relies on this
# useradd pcp

mkdir ~pcpserv && cd ~pcpserv
git clone https://github.com/performancecopilot/pcp
cd pcp
./Makepkgs
cd build/deb

# making a copy of modified configfiles..
cp /etc/pcp/netcheck/netcheck.conf /tmp
cp /etc/pcp/bind2/bind2.conf /tmp
rm -rf /tmp/config.d
cp -r /var/lib/pcp/pmdas/openmetrics/config.d/ /tmp/

# remove/install
apt-get remove libpcp-gui2 libpcp-import1 libpcp-pmda-perl \
  libpcp-mmv1 libpcp-pmda3 libpcp-trace2 libpcp-web1 libpcp3 \
  pcp pcp-conf python3-pcp
dpkg -i pcp_5.*_amd64.deb libpcp-gui2_5.*_amd64.deb \
  libpcp-import1_5.*_amd64.deb libpcp-mmv1_5.*_amd64.deb \
  libpcp-pmda-perl_*_amd64.deb \
  libpcp-pmda3_5.*_amd64.deb libpcp-trace2_5.*_amd64.deb \
  libpcp-web1_5.*_amd64.deb libpcp3_5.*_amd64.deb \
  pcp-conf_5.*_amd64.deb python3-pcp_5.*_amd64.deb

systemctl start pmcd
systemctl status pmcd

# restore old configs..
cd /var/lib/pcp/pmdas/postfix/ && ./Install
cp /tmp/netcheck.conf /etc/pcp/netcheck/netcheck.conf
cd /var/lib/pcp/pmdas/netcheck/ && ./Install
cp /tmp/bind2.conf /etc/pcp/bind2/
cd /var/lib/pcp/pmdas/bind2/ && ./Install

rm -f /var/lib/pcp/pmdas/openmetrics/config.d/*
cp /tmp/config.d/* /var/lib/pcp/pmdas/openmetrics/config.d/
cd /var/lib/pcp/pmdas/openmetrics/ && ./Install


# in my setup:
# systemctl restart pmcd pmlogger pmproxy grafana-server redis-server
systemctl enable --now pmproxy
pminfo bind2 netcheck openmetrics

Running the qa suite

# install required packages
yum localinstall pcp-testsuite pcp-pmda*rpm pcp-debuginfo*rpm

# preparing for QA tests
useradd -m pcpqa
echo 'pcpqa   ALL=(ALL) NOPASSWD: ALL' >/etc/sudoers.d/pcpqa
mkdir ~pcpqa/.ssh
cp /root/.ssh/authorized_keys ~pcpqa/.ssh
chown -R pcpqa:pcpqa ~pcpqa/.ssh

cd /var/lib/pcp/testsuite
./chk.setup
admin/check-vm
./check 000

$ cat ~/.gitconfig
[user]
email = chorn@<domain>
name = Christian Horn
[push]
default = simple

Own branches

# checkout own branch
git clone -b fix-lmsensors-parser \
  https://github.com/christianhorn/pcp.git fix-lmsensors-parser
  
# modify files
git commit .
git push

# pylint --output-format=colorized pmdalmsensors.python
# pylint --rcfile=/home/chris/Downloads/.pylintrc \
    --output-format=colorized pmdalmsensors.python

stripping down pmlogger metrics

You might just archive metrics with pmlogger for creating graphs with grafana, and not want to use them for debugging. In that case, you could

  • initially run pmlogger with the metrics suggested by pcp-zeroconf
  • then create dashboard which show everything you need. While doing that, you might add metrics to the pmlogger config, so they get archived.
  • once you are happy, you can use the dashboard json-code to understand which metrics you use, and create a new config which only contains _these_ metrics.

I used this workflow:

  • 'sudo dnf -y install xclip'
  • in your dashboard, go on the top to 'dashboard settings'
  • click 'json model', right click the json code, click 'select all' and 'copy'
# then, in a terminal use this to store the dashboard json
xclip -o -selection clipboard >grafana.json

# lets now extract the sources
grep expr grafana.json | sed -e 's,.*": ",\t,' -e 's,".*,,' -e 's,{.*,,'

Some values from there should be collapsed, for example
  	lmsensors.acpitz_acpi_0.temp1
	lmsensors.coretemp_isa_0000.core_0
	lmsensors.coretemp_isa_0000.core_1
into
        lmsensors
..which will capture all the deeper metrics.

# Then use the metrics in file
# /var/lib/pcp/config/pmlogger/config.default . For example:
######
log advisory on default {
	lmsensors
	kernel.all.load
	mem.util.cached
	mem.physmem
	mem.util.free
	kernel.percpu.cpu.user
	kernel.percpu.cpu.sys
	kernel.all.pswitch
	kernel.all.runnable
	mem.util.used
	mem.util.cached
	mem.util.free
	mem.vmstat.pgfault
	mem.vmstat.pgmajfault
	network.interface.in.bytes
	network.interface.out.bytes
	network.interface.in.drops
	network.interface.out.drops
	network.interface.in.packets
	network.interface.in.packets
	network.tcp.timeouts
	network.tcp.listenoverflows
	network.tcp.listendrops
	network.tcp.retranssegs
	network.tcp.fastretrans
	network.tcp.slowstartretrans
	network.tcp.synretrans
	disk.dev.read_rawactive
	disk.dev.write_rawactive
	disk.dev.read
	disk.dev.write
	disk.dev.read_bytes
	disk.dev.write_bytes
	disk.dev.avactive
}

[access]
disallow .* : all;
disallow :* : all;
allow local:* : enquire;
###

systemctl restart pmlogger

Hints

# Grafana shows only 1 week old data, but archive files with 2 weeks old data exist

# looking at grafana query:
http://127.0.0.1:3000/api/datasources/proxy/11/series/values?series=ed1fee80843
4b8c265752bd81ab18b4d8a45802c&start=1588916484&finish=1588916666&interval=0.01

# performing query manually
curl 'http://localhost:44322/series/values?series=ed1fee808434b8c265752bd81ab18b4d8a45802c&start=1588916484&finish=1588916666&interval=0.01'
[{"series":"ed1fee808434b8c265752bd81ab18b4d8a45802c","instance":"a2b4c26ae379f30881bfe0c82f6c30cc10d0934f","timestamp":1588916524035.758,"value":"4.284000e+00"}[..]

=> I always get "timestamp":1587801603901.84, and no newer timestamps.
Also querying for start=1000000000 does not change that

=> Verify with pmrep on commandline if older data is available:
pmrep -a 20200415.0 kernel.all.load -p -s 1000000000 \
  -f '%Y-%m-%d %H:%M:%S'|head
                     k.a.load  k.a.load  k.a.load
                     1 minute  5 minute  15 minut
                                                 
2020-04-15 00:10:07       N/A       N/A       N/A
2020-04-15 00:10:08       N/A       N/A       N/A
2020-04-15 00:10:09       N/A       N/A       N/A
2020-04-15 00:10:10       N/A       N/A       N/A
2020-04-15 00:10:11     0.070     0.140     0.160
2020-04-15 00:10:12     0.070     0.140     0.160
2020-04-15 00:10:13     0.070     0.140     0.160
=> Data is there..
software/performance_co-pilot/compiling.txt ยท Last modified: 2021/08/14 14:05 by chris