Commit 26538976 authored by Robert Schmidt's avatar Robert Schmidt

Merge remote-tracking branch 'origin/ci-exit-info-core-dumps' into integration_2024_w09

parents 54965235 56acd1d6
......@@ -5,6 +5,8 @@ services:
privileged: true
network_mode: "host"
container_name: oai-gnb
ulimits:
core: -1 # for core dumps
environment:
TZ: Europe/Paris
USE_ADDITIONAL_OPTIONS: --sa --tune-offset 20000000 -A 45 --log_config.global_log_options level,nocolor,time
......
......@@ -5,6 +5,8 @@ services:
privileged: true
network_mode: host
container_name: oai-nr-ue
ulimits:
core: -1 # for core dumps
#entrypoint: /bin/bash -c "sleep infinity"
environment:
TZ: Europe/Paris
......
......@@ -5,6 +5,8 @@ services:
image: oai-enb:latest
privileged: true
container_name: nsa-b200-enb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: --log_config.global_log_options level,nocolor,time,line_num,function
......
......@@ -5,6 +5,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: nsa-b200-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: -E -q --RUs.[0].sdr_addrs serial=30C51D4 --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function
......
......@@ -5,6 +5,8 @@ services:
privileged: true
network_mode: "host"
container_name: oai-gnb-aw2s
ulimits:
core: -1 # for core dumps
environment:
TZ: Europe/Paris
USE_ADDITIONAL_OPTIONS: --sa
......
......@@ -5,6 +5,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-b200-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function
......
......@@ -5,6 +5,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-cucp-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_ADDITIONAL_OPTIONS: --sa --telnetsrv --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function
volumes:
......@@ -23,6 +25,8 @@ services:
image: oai-nr-cuup:latest
privileged: true
container_name: sa-cuup-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_ADDITIONAL_OPTIONS: --sa --log_config.global_log_options level,nocolor,time,line_num,function
volumes:
......@@ -41,6 +45,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-du-b200-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: --sa
......
......@@ -5,6 +5,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-cu-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_ADDITIONAL_OPTIONS: --sa --telnetsrv --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function
volumes:
......@@ -24,6 +26,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-du-b200-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --log_config.global_log_options level,nocolor,time,line_num,function --gNBs.[0].min_rxtxtime 2 --gNBs.[0].do_CSIRS 1 --gNBs.[0].do_SRS 0 --MACRLCs.[0].ul_max_mcs 28 --L1s.[0].max_ldpc_iterations 20
......
......@@ -5,6 +5,8 @@ services:
image: oai-gnb:latest
privileged: true
container_name: sa-b200-gnb
ulimits:
core: -1 # for core dumps
environment:
USE_B2XX: 'yes'
USE_ADDITIONAL_OPTIONS: --sa --RUs.[0].sdr_addrs serial=30C51D4 --telnetsrv --telnetsrv.shrmod ci --continuous-tx --log_config.global_log_options level,nocolor,time,line_num,function -E
......
......@@ -249,3 +249,71 @@ steps](../docker/README.md) and then use the docker-compose file directly.
Some tests are run from source (e.g.
`ci-scripts/xml_files/gnb_phytest_usrp_run.xml`), which directly give the
options they are run with.
## How to retrieve core dumps (for CI team members)
The entrypoint scripts of all containers print the core pattern that is used on
the running machine. Search for `core_pattern` at the start of the container
logs to retrieve the possible location. Possible locations might be:
- a path: the corresponding directory must be mounted in the container to be
writable
- systemd-coredumpd: see [documentation](https://systemd.io/COREDUMP/)
- abrt: see [documentation](https://abrt.readthedocs.io/en/latest/usage.html)
- apport: see [documentation](https://wiki.ubuntu.com/Apport)
You furthermore have to extract the executable that caused the core dump.
Download the container image, and extract, e.g.:
```
docker create --name c1 porcepix.sboai.cs.eurecom.fr/oai-gnb:develop-c99db698
docker cp c1:/opt/oai-gnb/bin/nr-softmodem /tmp
docker rm c1
```
### Core dump in a file
**This is not recommended, as files could pile up and fill the system disk
completely!** Prefer systemd or abrt instead.
If the core pattern is a path: it should at least include the time in the
pattern name (suggested pattern: `/tmp/core.%e.%p.%t`) to correlate the time
the segfault occurred with the CI logs. If you identified the core dump,
copy the core dump from that machine; if identification is difficult, consider
rerunning the pipeline.
### Core dump via systemd
Run this command to list all core dumps:
```
sudo coredumpctl list
```
Scroll to the end and find the core dump of interest (it lists the executables
in the last column; use the time to correlate the segfault and the CI run).
Take the PID of the executable (first column after the time). Dump the core
dump to a location of your choice:
```
sudo coredumpctl dump <PID> > /tmp/coredump
```
### Core dump via abrt (automatic bug reporting tool)
TBD: use the documentation page for the moment.
### Core dump via apport
On Ubuntu machines, apport first needs to be enabled to collect core dumps:
```
sudo systemctl enable apport.service
```
and [needs to be enabled](https://wiki.ubuntu.com/Apport#How_to_enable_apport).
Then, show a list of core dumps using
```
sudo apport-cli
```
......@@ -58,6 +58,12 @@ RUN rm -f /etc/rhsm-host && \
echo "/usr/local/lib" > /etc/ld.so.conf.d/local-lib.conf && \
echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local-lib.conf
# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
# it will be copied into target containers, to print exit numbers and handle signals properly
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
# In some network environments, GIT proxy is required
RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi"
......
......@@ -46,6 +46,12 @@ RUN dnf install 'dnf-command(config-manager)' -y && \
python3-pip && \
pip3 install --ignore-installed pyyaml
# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
# it will be copied into target containers, to print exit numbers and handle signals properly
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
#create the WORKDIR
WORKDIR /oai-ran
COPY . .
......
......@@ -46,6 +46,12 @@ RUN apt-get update && \
python3-pip && \
pip3 install --ignore-installed pyyaml
# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
# it will be copied into target containers, to print exit numbers and handle signals properly
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
# In some network environments, GIT proxy is required
RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi"
......
......@@ -73,6 +73,12 @@ RUN apt-get update && \
g++-9-aarch64-linux-gnu && \
apt-get clean
# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
# it will be copied into target containers, to print exit numbers and handle signals properly
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
# create the WORKDIR
WORKDIR /oai-ran
COPY . .
......
......@@ -112,5 +112,6 @@ WORKDIR /opt/oai-enb
# 36422 --> X2C, SCTP/UDP
EXPOSE 2152/udp 36412/udp 36422/udp
ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"]
COPY --from=enb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"]
CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"]
......@@ -122,5 +122,6 @@ WORKDIR /opt/oai-enb
# 36422 --> X2C, SCTP/UDP
EXPOSE 2152/udp 36412/udp 36422/udp
ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"]
COPY --from=enb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"]
CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"]
......@@ -112,5 +112,6 @@ EXPOSE 2152/udp 36412/udp 36422/udp
#EXPOSE 50000/udp # IF5 / ORI (control)
#EXPOSE 50001/udp # IF5 / ECPRI (data)
ENTRYPOINT ["/opt/oai-enb/bin/entrypoint.sh"]
COPY --from=enb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-enb/bin/entrypoint.sh"]
CMD ["/opt/oai-enb/bin/lte-softmodem", "-O", "/opt/oai-enb/etc/enb.conf"]
......@@ -86,5 +86,6 @@ RUN ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/libthirdparty_tra
WORKDIR /opt/oai-gnb-aw2s
ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"]
......@@ -95,5 +95,6 @@ RUN ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/libthirdparty_tra
WORKDIR /opt/oai-gnb-aw2s
ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"]
......@@ -81,5 +81,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/libaw2sori_transpro.so /usr/local/lib/lib
WORKDIR /opt/oai-gnb-aw2s
ENTRYPOINT ["/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb-aw2s/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb-aw2s/bin/nr-softmodem", "-O", "/opt/oai-gnb-aw2s/etc/gnb.conf"]
......@@ -111,5 +111,6 @@ WORKDIR /opt/oai-gnb
#EXPOSE 50000/udp # IF5 / ORI (control)
#EXPOSE 50001/udp # IF5 / ECPRI (data)
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -121,5 +121,6 @@ WORKDIR /opt/oai-gnb
#EXPOSE 50000/udp # IF5 / ORI (control)
#EXPOSE 50001/udp # IF5 / ECPRI (data)
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -108,5 +108,6 @@ WORKDIR /opt/oai-gnb
#EXPOSE 50000/udp # IF5 / ORI (control)
#EXPOSE 50001/udp # IF5 / ECPRI (data)
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-softmodem", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -95,5 +95,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
WORKDIR /opt/oai-lte-ru
ENTRYPOINT ["/opt/oai-lte-ru/bin/entrypoint.sh"]
COPY --from=ru-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ru/bin/entrypoint.sh"]
CMD ["/opt/oai-lte-ru/bin/oairu", "-O", "/opt/oai-lte-ru/etc/rru.conf"]
......@@ -88,5 +88,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
WORKDIR /opt/oai-lte-ru
ENTRYPOINT ["/opt/oai-lte-ru/bin/entrypoint.sh"]
COPY --from=ru-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ru/bin/entrypoint.sh"]
CMD ["/opt/oai-lte-ru/bin/oairu", "-O", "/opt/oai-lte-ru/etc/rru.conf"]
......@@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
ldconfig
WORKDIR /opt/oai-lte-ue
COPY --from=lte-ue-base /tini /tini
CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"]
ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"]
......@@ -116,5 +116,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
echo "ldd on libtelnetsrv.so" && ldd /usr/local/lib/libtelnetsrv.so
WORKDIR /opt/oai-lte-ue
COPY --from=lte-ue-base /tini /tini
CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"]
ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"]
......@@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
ldd /opt/oai-lte-ue/bin/lte-uesoftmodem
WORKDIR /opt/oai-lte-ue
COPY --from=lte-ue-base /tini /tini
CMD ["/opt/oai-lte-ue/bin/lte-uesoftmodem"]
ENTRYPOINT ["/opt/oai-lte-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-lte-ue/bin/entrypoint.sh"]
......@@ -75,5 +75,6 @@ WORKDIR /opt/oai-gnb/etc
WORKDIR /opt/oai-gnb
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -76,5 +76,6 @@ WORKDIR /opt/oai-gnb/etc
WORKDIR /opt/oai-gnb
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -66,5 +66,6 @@ RUN ldconfig && ldd /opt/oai-gnb/bin/nr-cuup
WORKDIR /opt/oai-gnb/etc
WORKDIR /opt/oai-gnb
ENTRYPOINT ["/opt/oai-gnb/bin/entrypoint.sh"]
COPY --from=gnb-base /tini /tini
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
CMD ["/opt/oai-gnb/bin/nr-cuup", "-O", "/opt/oai-gnb/etc/gnb.conf"]
......@@ -108,5 +108,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
WORKDIR /opt/oai-nr-ue
COPY --from=nr-ue-base /tini /tini
CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"]
ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]
......@@ -117,5 +117,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
echo "ldd on libtelnetsrv_5Gue.so" && ldd /usr/local/lib/libtelnetsrv_5Gue.so
WORKDIR /opt/oai-nr-ue
COPY --from=nr-ue-base /tini /tini
CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"]
ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]
......@@ -104,5 +104,6 @@ RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai
ldd /opt/oai-nr-ue/bin/nr-uesoftmodem
WORKDIR /opt/oai-nr-ue
COPY --from=nr-ue-base /tini /tini
CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem", "-O", "/opt/oai-nr-ue/etc/nr-ue.conf"]
ENTRYPOINT ["/opt/oai-nr-ue/bin/entrypoint.sh"]
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]
......@@ -5,6 +5,9 @@ set -uo pipefail
PREFIX=/opt/oai-enb
CONFIGFILE=$PREFIX/etc/enb.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $CONFIGFILE ]; then
echo "No configuration file found: please mount at $CONFIGFILE"
exit 255
......
......@@ -5,6 +5,9 @@ set -uo pipefail
PREFIX=/opt/oai-gnb-aw2s
CONFIGFILE=$PREFIX/etc/gnb.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $CONFIGFILE ]; then
echo "No configuration file found: please mount at $CONFIGFILE"
exit 255
......
......@@ -5,6 +5,8 @@ set -uo pipefail
PREFIX=/opt/oai-gnb
CONFIGFILE=$PREFIX/etc/gnb.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $CONFIGFILE ]; then
echo "No configuration file found: please mount at $CONFIGFILE"
......
......@@ -5,6 +5,9 @@ set -uo pipefail
PREFIX=/opt/oai-lte-ru
CONFIGFILE=$PREFIX/etc/rru.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $CONFIGFILE ]; then
echo "No configuration file found: please mount at $CONFIGFILE"
exit 255
......
......@@ -5,6 +5,9 @@ set -uo pipefail
PREFIX=/opt/oai-lte-ue
USIM_CONFIGFILE=$PREFIX/etc/ue_usim.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $USIM_CONFIGFILE ]; then
echo "No ue_usim.conf configuration file found: please mount at $USIM_CONFIGFILE"
exit 255
......
......@@ -5,6 +5,9 @@ set -uo pipefail
PREFIX=/opt/oai-nr-ue
CONFIGFILE=$PREFIX/etc/nr-ue.conf
echo "=================================="
echo "/proc/sys/kernel/core_pattern=$(cat /proc/sys/kernel/core_pattern)"
if [ ! -f $CONFIGFILE ]; then
echo "No configuration file found: please mount at $CONFIGFILE"
exit 255
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment