diff options
Diffstat (limited to 'kernel/Documentation/mic/mpssd')
| -rw-r--r-- | kernel/Documentation/mic/mpssd/Makefile | 2 | ||||
| -rwxr-xr-x | kernel/Documentation/mic/mpssd/mpss | 28 | ||||
| -rw-r--r-- | kernel/Documentation/mic/mpssd/mpssd.c | 362 | ||||
| -rw-r--r-- | kernel/Documentation/mic/mpssd/mpssd.h | 1 |
4 files changed, 246 insertions, 147 deletions
diff --git a/kernel/Documentation/mic/mpssd/Makefile b/kernel/Documentation/mic/mpssd/Makefile index f47fe6ba7..06871b0c0 100644 --- a/kernel/Documentation/mic/mpssd/Makefile +++ b/kernel/Documentation/mic/mpssd/Makefile @@ -1,3 +1,4 @@ +ifndef CROSS_COMPILE # List of programs to build hostprogs-$(CONFIG_X86_64) := mpssd @@ -17,3 +18,4 @@ HOSTLOADLIBES_mpssd := -lpthread install: install mpssd /usr/sbin/mpssd install micctrl /usr/sbin/micctrl +endif diff --git a/kernel/Documentation/mic/mpssd/mpss b/kernel/Documentation/mic/mpssd/mpss index cacbdb0ae..09ea90931 100755 --- a/kernel/Documentation/mic/mpssd/mpss +++ b/kernel/Documentation/mic/mpssd/mpss @@ -35,6 +35,7 @@ exec=/usr/sbin/mpssd sysfs="/sys/class/mic" +mic_modules="mic_host mic_x100_dma scif" start() { @@ -48,18 +49,15 @@ start() fi echo -e $"Starting MPSS Stack" - echo -e $"Loading MIC_X100_DMA & MIC_HOST Modules" + echo -e $"Loading MIC drivers:" $mic_modules - for f in "mic_host" "mic_x100_dma" - do - modprobe $f - RETVAL=$? - if [ $RETVAL -ne 0 ]; then - failure - echo - return $RETVAL - fi - done + modprobe -a $mic_modules + RETVAL=$? + if [ $RETVAL -ne 0 ]; then + failure + echo + return $RETVAL + fi # Start the daemon echo -n $"Starting MPSSD " @@ -121,10 +119,10 @@ stop() # Wait for the cards to go offline for f in $sysfs/* do - while [ "`cat $f/state`" != "offline" ] + while [ "`cat $f/state`" != "ready" ] do sleep 1 - echo -e "Waiting for "`basename $f`" to go offline" + echo -e "Waiting for "`basename $f`" to become ready" done done @@ -170,8 +168,8 @@ unload() stop sleep 5 - echo -n $"Removing MIC_HOST & MIC_X100_DMA Modules: " - modprobe -r mic_host mic_x100_dma + echo -n $"Removing MIC drivers:" $mic_modules + modprobe -r $mic_modules RETVAL=$? [ $RETVAL -ne 0 ] && failure || success echo diff --git a/kernel/Documentation/mic/mpssd/mpssd.c b/kernel/Documentation/mic/mpssd/mpssd.c index 3c5c379fc..aaeafa18d 100644 --- a/kernel/Documentation/mic/mpssd/mpssd.c +++ b/kernel/Documentation/mic/mpssd/mpssd.c @@ -43,7 +43,7 @@ #include <linux/mic_common.h> #include <tools/endian.h> -static void init_mic(struct mic_info *mic); +static void *init_mic(void *arg); static FILE *logfp; static struct mic_info mic_list; @@ -116,19 +116,18 @@ static struct { .num = htole16(MIC_VRING_ENTRIES), }, #if GSO_ENABLED - .host_features = htole32( + .host_features = htole32( 1 << VIRTIO_NET_F_CSUM | 1 << VIRTIO_NET_F_GSO | 1 << VIRTIO_NET_F_GUEST_TSO4 | 1 << VIRTIO_NET_F_GUEST_TSO6 | - 1 << VIRTIO_NET_F_GUEST_ECN | - 1 << VIRTIO_NET_F_GUEST_UFO), + 1 << VIRTIO_NET_F_GUEST_ECN), #else .host_features = 0, #endif }; -static const char *mic_config_dir = "/etc/sysconfig/mic"; +static const char *mic_config_dir = "/etc/mpss"; static const char *virtblk_backend = "VIRTBLK_BACKEND"; static struct { struct mic_device_desc dd; @@ -192,7 +191,7 @@ tap_configure(struct mic_info *mic, char *dev) return ret; } - snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id); + snprintf(ipaddr, IFNAMSIZ, "172.31.%d.254/24", mic->id + 1); pid = fork(); if (pid == 0) { @@ -255,8 +254,7 @@ static int tun_alloc(struct mic_info *mic, char *dev) return err; } #if GSO_ENABLED - offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | - TUN_F_TSO_ECN | TUN_F_UFO; + offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_TSO_ECN; err = ioctl(fd, TUNSETOFFLOAD, offload); if (err < 0) { @@ -332,7 +330,6 @@ static struct mic_device_desc *get_device_desc(struct mic_info *mic, int type) return d; } mpsslog("%s %s %d not found\n", mic->name, __func__, type); - assert(0); return NULL; } @@ -415,6 +412,13 @@ mic_virtio_copy(struct mic_info *mic, int fd, return ret; } +static inline unsigned _vring_size(unsigned int num, unsigned long align) +{ + return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num) + + align - 1) & ~(align - 1)) + + sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num; +} + /* * This initialization routine requires at least one * vring i.e. vr0. vr1 is optional. @@ -426,8 +430,9 @@ init_vr(struct mic_info *mic, int fd, int type, int vr_size; char *va; - vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES, - MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info)); + vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES, + MIC_VIRTIO_RING_ALIGN) + + sizeof(struct _mic_vring_info)); va = mmap(NULL, MIC_DEVICE_PAGE_END + vr_size * num_vq, PROT_READ, MAP_SHARED, fd, 0); if (MAP_FAILED == va) { @@ -439,25 +444,25 @@ init_vr(struct mic_info *mic, int fd, int type, set_dp(mic, type, va); vr0->va = (struct mic_vring *)&va[MIC_DEVICE_PAGE_END]; vr0->info = vr0->va + - vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN); + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN); vring_init(&vr0->vr, MIC_VRING_ENTRIES, vr0->va, MIC_VIRTIO_RING_ALIGN); mpsslog("%s %s vr0 %p vr0->info %p vr_size 0x%x vring 0x%x ", __func__, mic->name, vr0->va, vr0->info, vr_size, - vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN)); + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN)); mpsslog("magic 0x%x expected 0x%x\n", le32toh(vr0->info->magic), MIC_MAGIC + type); assert(le32toh(vr0->info->magic) == MIC_MAGIC + type); if (vr1) { vr1->va = (struct mic_vring *) &va[MIC_DEVICE_PAGE_END + vr_size]; - vr1->info = vr1->va + vring_size(MIC_VRING_ENTRIES, + vr1->info = vr1->va + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN); vring_init(&vr1->vr, MIC_VRING_ENTRIES, vr1->va, MIC_VIRTIO_RING_ALIGN); mpsslog("%s %s vr1 %p vr1->info %p vr_size 0x%x vring 0x%x ", __func__, mic->name, vr1->va, vr1->info, vr_size, - vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN)); + _vring_size(MIC_VRING_ENTRIES, MIC_VIRTIO_RING_ALIGN)); mpsslog("magic 0x%x expected 0x%x\n", le32toh(vr1->info->magic), MIC_MAGIC + type + 1); assert(le32toh(vr1->info->magic) == MIC_MAGIC + type + 1); @@ -466,16 +471,21 @@ done: return va; } -static void +static int wait_for_card_driver(struct mic_info *mic, int fd, int type) { struct pollfd pollfd; int err; struct mic_device_desc *desc = get_device_desc(mic, type); + __u8 prev_status; + if (!desc) + return -ENODEV; + prev_status = desc->status; pollfd.fd = fd; mpsslog("%s %s Waiting .... desc-> type %d status 0x%x\n", mic->name, __func__, type, desc->status); + while (1) { pollfd.events = POLLIN; pollfd.revents = 0; @@ -487,8 +497,13 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type) } if (pollfd.revents) { - mpsslog("%s %s Waiting... desc-> type %d status 0x%x\n", - mic->name, __func__, type, desc->status); + if (desc->status != prev_status) { + mpsslog("%s %s Waiting... desc-> type %d " + "status 0x%x\n", + mic->name, __func__, type, + desc->status); + prev_status = desc->status; + } if (desc->status & VIRTIO_CONFIG_S_DRIVER_OK) { mpsslog("%s %s poll.revents %d\n", mic->name, __func__, pollfd.revents); @@ -499,6 +514,7 @@ wait_for_card_driver(struct mic_info *mic, int fd, int type) } } } + return 0; } /* Spin till we have some descriptors */ @@ -575,9 +591,16 @@ virtio_net(void *arg) __func__, strerror(errno)); continue; } - if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) - wait_for_card_driver(mic, mic->mic_net.virtio_net_fd, - VIRTIO_ID_NET); + if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) { + err = wait_for_card_driver(mic, + mic->mic_net.virtio_net_fd, + VIRTIO_ID_NET); + if (err) { + mpsslog("%s %s %d Exiting...\n", + mic->name, __func__, __LINE__); + break; + } + } /* * Check if there is data to be read from TUN and write to * virtio net fd if there is. @@ -786,10 +809,16 @@ virtio_console(void *arg) strerror(errno)); continue; } - if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) - wait_for_card_driver(mic, - mic->mic_console.virtio_console_fd, - VIRTIO_ID_CONSOLE); + if (!(desc->status & VIRTIO_CONFIG_S_DRIVER_OK)) { + err = wait_for_card_driver(mic, + mic->mic_console.virtio_console_fd, + VIRTIO_ID_CONSOLE); + if (err) { + mpsslog("%s %s %d Exiting...\n", + mic->name, __func__, __LINE__); + break; + } + } if (console_poll[MONITOR_FD].revents & POLLIN) { copy.iov = iov0; @@ -1048,8 +1077,9 @@ stop_virtblk(struct mic_info *mic) { int vr_size, ret; - vr_size = PAGE_ALIGN(vring_size(MIC_VRING_ENTRIES, - MIC_VIRTIO_RING_ALIGN) + sizeof(struct _mic_vring_info)); + vr_size = PAGE_ALIGN(_vring_size(MIC_VRING_ENTRIES, + MIC_VIRTIO_RING_ALIGN) + + sizeof(struct _mic_vring_info)); ret = munmap(mic->mic_virtblk.block_dp, MIC_DEVICE_PAGE_END + vr_size * virtblk_dev_page.dd.num_vq); if (ret < 0) @@ -1131,6 +1161,10 @@ write_status(int fd, __u8 *status) return ioctl(fd, MIC_VIRTIO_COPY_DESC, ©); } +#ifndef VIRTIO_BLK_T_GET_ID +#define VIRTIO_BLK_T_GET_ID 8 +#endif + static void * virtio_block(void *arg) { @@ -1297,12 +1331,7 @@ reset(struct mic_info *mic) mpsslog("%s: %s %d state %s\n", mic->name, __func__, __LINE__, state); - /* - * If the shutdown was initiated by OSPM, the state stays - * in "suspended" which is also a valid condition for reset. - */ - if ((!strcmp(state, "offline")) || - (!strcmp(state, "suspended"))) { + if (!strcmp(state, "ready")) { free(state); break; } @@ -1331,34 +1360,50 @@ get_mic_shutdown_status(struct mic_info *mic, char *shutdown_status) assert(0); }; -static int get_mic_state(struct mic_info *mic, char *state) +static int get_mic_state(struct mic_info *mic) { - if (!strcmp(state, "offline")) - return MIC_OFFLINE; - if (!strcmp(state, "online")) - return MIC_ONLINE; - if (!strcmp(state, "shutting_down")) - return MIC_SHUTTING_DOWN; - if (!strcmp(state, "reset_failed")) - return MIC_RESET_FAILED; - if (!strcmp(state, "suspending")) - return MIC_SUSPENDING; - if (!strcmp(state, "suspended")) - return MIC_SUSPENDED; - mpsslog("%s: BUG invalid state %s\n", mic->name, state); - /* Invalid state */ - assert(0); + char *state = NULL; + enum mic_states mic_state; + + while (!state) { + state = readsysfs(mic->name, "state"); + sleep(1); + } + mpsslog("%s: %s %d state %s\n", + mic->name, __func__, __LINE__, state); + + if (!strcmp(state, "ready")) { + mic_state = MIC_READY; + } else if (!strcmp(state, "booting")) { + mic_state = MIC_BOOTING; + } else if (!strcmp(state, "online")) { + mic_state = MIC_ONLINE; + } else if (!strcmp(state, "shutting_down")) { + mic_state = MIC_SHUTTING_DOWN; + } else if (!strcmp(state, "reset_failed")) { + mic_state = MIC_RESET_FAILED; + } else if (!strcmp(state, "resetting")) { + mic_state = MIC_RESETTING; + } else { + mpsslog("%s: BUG invalid state %s\n", mic->name, state); + assert(0); + } + + free(state); + return mic_state; }; static void mic_handle_shutdown(struct mic_info *mic) { #define SHUTDOWN_TIMEOUT 60 - int i = SHUTDOWN_TIMEOUT, ret, stat = 0; + int i = SHUTDOWN_TIMEOUT; char *shutdown_status; while (i) { shutdown_status = readsysfs(mic->name, "shutdown_status"); - if (!shutdown_status) + if (!shutdown_status) { + sleep(1); continue; + } mpsslog("%s: %s %d shutdown_status %s\n", mic->name, __func__, __LINE__, shutdown_status); switch (get_mic_shutdown_status(mic, shutdown_status)) { @@ -1377,94 +1422,110 @@ static void mic_handle_shutdown(struct mic_info *mic) i--; } reset: - ret = kill(mic->pid, SIGTERM); - mpsslog("%s: %s %d kill pid %d ret %d\n", - mic->name, __func__, __LINE__, - mic->pid, ret); - if (!ret) { - ret = waitpid(mic->pid, &stat, - WIFSIGNALED(stat)); - mpsslog("%s: %s %d waitpid ret %d pid %d\n", - mic->name, __func__, __LINE__, - ret, mic->pid); - } - if (ret == mic->pid) - reset(mic); + if (!i) + mpsslog("%s: %s %d timing out waiting for shutdown_status %s\n", + mic->name, __func__, __LINE__, shutdown_status); + reset(mic); } -static void * -mic_config(void *arg) +static int open_state_fd(struct mic_info *mic) { - struct mic_info *mic = (struct mic_info *)arg; - char *state = NULL; char pathname[PATH_MAX]; - int fd, ret; - struct pollfd ufds[1]; - char value[4096]; + int fd; snprintf(pathname, PATH_MAX - 1, "%s/%s/%s", MICSYSFSDIR, mic->name, "state"); fd = open(pathname, O_RDONLY); - if (fd < 0) { + if (fd < 0) mpsslog("%s: opening file %s failed %s\n", mic->name, pathname, strerror(errno)); - goto error; + return fd; +} + +static int block_till_state_change(int fd, struct mic_info *mic) +{ + struct pollfd ufds[1]; + char value[PAGE_SIZE]; + int ret; + + ufds[0].fd = fd; + ufds[0].events = POLLERR | POLLPRI; + ret = poll(ufds, 1, -1); + if (ret < 0) { + mpsslog("%s: %s %d poll failed %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + return ret; + } + + ret = lseek(fd, 0, SEEK_SET); + if (ret < 0) { + mpsslog("%s: %s %d Failed to seek to 0: %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + return ret; + } + + ret = read(fd, value, sizeof(value)); + if (ret < 0) { + mpsslog("%s: %s %d Failed to read sysfs entry: %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + return ret; + } + + return 0; +} + +static void * +mic_config(void *arg) +{ + struct mic_info *mic = (struct mic_info *)arg; + int fd, ret, stat = 0; + + fd = open_state_fd(mic); + if (fd < 0) { + mpsslog("%s: %s %d open state fd failed %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + goto exit; } do { - ret = lseek(fd, 0, SEEK_SET); + ret = block_till_state_change(fd, mic); if (ret < 0) { - mpsslog("%s: Failed to seek to file start '%s': %s\n", - mic->name, pathname, strerror(errno)); - goto close_error1; + mpsslog("%s: %s %d block_till_state_change error %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + goto close_exit; } - ret = read(fd, value, sizeof(value)); - if (ret < 0) { - mpsslog("%s: Failed to read sysfs entry '%s': %s\n", - mic->name, pathname, strerror(errno)); - goto close_error1; - } -retry: - state = readsysfs(mic->name, "state"); - if (!state) - goto retry; - mpsslog("%s: %s %d state %s\n", - mic->name, __func__, __LINE__, state); - switch (get_mic_state(mic, state)) { + + switch (get_mic_state(mic)) { case MIC_SHUTTING_DOWN: mic_handle_shutdown(mic); - goto close_error; - case MIC_SUSPENDING: - mic->boot_on_resume = 1; - setsysfs(mic->name, "state", "suspend"); - mic_handle_shutdown(mic); - goto close_error; - case MIC_OFFLINE: + break; + case MIC_READY: + case MIC_RESET_FAILED: + ret = kill(mic->pid, SIGTERM); + mpsslog("%s: %s %d kill pid %d ret %d\n", + mic->name, __func__, __LINE__, + mic->pid, ret); + if (!ret) { + ret = waitpid(mic->pid, &stat, + WIFSIGNALED(stat)); + mpsslog("%s: %s %d waitpid ret %d pid %d\n", + mic->name, __func__, __LINE__, + ret, mic->pid); + } if (mic->boot_on_resume) { setsysfs(mic->name, "state", "boot"); mic->boot_on_resume = 0; } - break; + goto close_exit; default: break; } - free(state); - - ufds[0].fd = fd; - ufds[0].events = POLLERR | POLLPRI; - ret = poll(ufds, 1, -1); - if (ret < 0) { - mpsslog("%s: poll failed %s\n", - mic->name, strerror(errno)); - goto close_error1; - } } while (1); -close_error: - free(state); -close_error1: + +close_exit: close(fd); -error: +exit: init_mic(mic); pthread_exit(NULL); } @@ -1477,15 +1538,15 @@ set_cmdline(struct mic_info *mic) len = snprintf(buffer, PATH_MAX, "clocksource=tsc highres=off nohz=off "); - len += snprintf(buffer + len, PATH_MAX - len, + len += snprintf(buffer + len, PATH_MAX, "cpufreq_on;corec6_off;pc3_off;pc6_off "); - len += snprintf(buffer + len, PATH_MAX - len, + len += snprintf(buffer + len, PATH_MAX, "ifcfg=static;address,172.31.%d.1;netmask,255.255.255.0", - mic->id); + mic->id + 1); setsysfs(mic->name, "cmdline", buffer); mpsslog("%s: Command line: \"%s\"\n", mic->name, buffer); - snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id); + snprintf(buffer, PATH_MAX, "172.31.%d.1", mic->id + 1); mpsslog("%s: IPADDR: \"%s\"\n", mic->name, buffer); } @@ -1541,8 +1602,6 @@ set_log_buf_info(struct mic_info *mic) close(fd); } -static void init_mic(struct mic_info *mic); - static void change_virtblk_backend(int x, siginfo_t *siginfo, void *p) { @@ -1553,8 +1612,16 @@ change_virtblk_backend(int x, siginfo_t *siginfo, void *p) } static void -init_mic(struct mic_info *mic) +set_mic_boot_params(struct mic_info *mic) +{ + set_log_buf_info(mic); + set_cmdline(mic); +} + +static void * +init_mic(void *arg) { + struct mic_info *mic = (struct mic_info *)arg; struct sigaction ignore = { .sa_flags = 0, .sa_handler = SIG_IGN @@ -1564,7 +1631,7 @@ init_mic(struct mic_info *mic) .sa_sigaction = change_virtblk_backend, }; char buffer[PATH_MAX]; - int err; + int err, fd; /* * Currently, one virtio block device is supported for each MIC card @@ -1577,12 +1644,38 @@ init_mic(struct mic_info *mic) * the MIC daemon. */ sigaction(SIGUSR1, &ignore, NULL); +retry: + fd = open_state_fd(mic); + if (fd < 0) { + mpsslog("%s: %s %d open state fd failed %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + sleep(2); + goto retry; + } + + if (mic->restart) { + snprintf(buffer, PATH_MAX, "boot"); + setsysfs(mic->name, "state", buffer); + mpsslog("%s restarting mic %d\n", + mic->name, mic->restart); + mic->restart = 0; + } + + while (1) { + while (block_till_state_change(fd, mic)) { + mpsslog("%s: %s %d block_till_state_change error %s\n", + mic->name, __func__, __LINE__, strerror(errno)); + sleep(2); + continue; + } + + if (get_mic_state(mic) == MIC_BOOTING) + break; + } mic->pid = fork(); switch (mic->pid) { case 0: - set_log_buf_info(mic); - set_cmdline(mic); add_virtio_device(mic, &virtcons_dev_page.dd); add_virtio_device(mic, &virtnet_dev_page.dd); err = pthread_create(&mic->mic_console.console_thread, NULL, @@ -1612,24 +1705,29 @@ init_mic(struct mic_info *mic) mic->name, mic->id, errno); break; default: - if (mic->restart) { - snprintf(buffer, PATH_MAX, "boot"); - setsysfs(mic->name, "state", buffer); - mpsslog("%s restarting mic %d\n", - mic->name, mic->restart); - mic->restart = 0; - } - pthread_create(&mic->config_thread, NULL, mic_config, mic); + err = pthread_create(&mic->config_thread, NULL, + mic_config, mic); + if (err) + mpsslog("%s mic_config pthread_create failed %s\n", + mic->name, strerror(err)); } + + return NULL; } static void start_daemon(void) { struct mic_info *mic; + int err; - for (mic = mic_list.next; mic != NULL; mic = mic->next) - init_mic(mic); + for (mic = mic_list.next; mic; mic = mic->next) { + set_mic_boot_params(mic); + err = pthread_create(&mic->init_thread, NULL, init_mic, mic); + if (err) + mpsslog("%s init_mic pthread_create failed %s\n", + mic->name, strerror(err)); + } while (1) sleep(60); diff --git a/kernel/Documentation/mic/mpssd/mpssd.h b/kernel/Documentation/mic/mpssd/mpssd.h index f5f18b15d..8bd64944a 100644 --- a/kernel/Documentation/mic/mpssd/mpssd.h +++ b/kernel/Documentation/mic/mpssd/mpssd.h @@ -86,6 +86,7 @@ struct mic_info { int id; char *name; pthread_t config_thread; + pthread_t init_thread; pid_t pid; struct mic_console_info mic_console; struct mic_net_info mic_net; |